diff --git a/.gitattributes b/.gitattributes index 6a524f5eda739e1ad924ca55e94f2d30c7732aa1..f18d884c691143eaa0fce45106ac04c9b63ece5e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,7 +10,6 @@ *.lfs.* filter=lfs diff=lfs merge=lfs -text *.mlmodel filter=lfs diff=lfs merge=lfs -text *.model filter=lfs diff=lfs merge=lfs -text -*.axmodel filter=lfs diff=lfs merge=lfs -text *.msgpack filter=lfs diff=lfs merge=lfs -text *.npy filter=lfs diff=lfs merge=lfs -text *.npz filter=lfs diff=lfs merge=lfs -text @@ -34,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +models/630C/openwakeword__alexa_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/630C/openwakeword__embedding_model.axmodel filter=lfs diff=lfs merge=lfs -text +models/630C/openwakeword__hey_jarvis_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/630C/openwakeword__hey_mycroft_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/630C/openwakeword__hey_rhasspy_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/630C/openwakeword__melspectrogram.axmodel filter=lfs diff=lfs merge=lfs -text +models/630C/openwakeword__timer_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/630C/openwakeword__weather_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/650/openwakeword__alexa_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/650/openwakeword__embedding_model.axmodel filter=lfs diff=lfs merge=lfs -text +models/650/openwakeword__hey_jarvis_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/650/openwakeword__hey_mycroft_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/650/openwakeword__hey_rhasspy_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/650/openwakeword__melspectrogram.axmodel filter=lfs diff=lfs merge=lfs -text +models/650/openwakeword__timer_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text +models/650/openwakeword__weather_v0.1.axmodel filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 32897cd3e640101ba184f8c4ccd896981de3804a..1649fde50146bda0745b1e4ee79775f3d5739a7f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,125 @@ ---- -license: mit ---- +# openWakeWord.AXERA + +[openWakeWord](https://github.com/dscripka/openWakeWord) 语音唤醒 AXERA 推理 demo。 + +## 功能 + +- 支持 AX650 和 AX630C； +- 支持 Python 和 C++ 板端推理； + +## 目录结构 + +```text +OpenWakeWord.AXERA/ +├── models/ +│ ├── 650/ # AX650 axmodel +│ └── 630C/ # AX630C axmodel +├── config/ # STFT/mel 权重 +├── audio/openwakeword/ +├── scripts/ # Python 推理代码 +├── cpp/ # C++ 源码、编译和运行脚本 +└── README.md +``` + + +## Python + +```bash +创建虚拟环境 +conda create -n OpenWakeWord python=3.10 +conda activate OpenWakeWord + +安装 [pyaxengine](https://github.com/AXERA-TECH/pyaxengine/releases/latest)： +pip install axengine-x.x.x-py3-none-any.whl + +pip install -r requirements.txt +``` + +AX650： + +```bash +python scripts/openwakeword_ax650.py +``` + +AX630C： + +```bash +python scripts/openwakeword_ax630c.py +``` + +自定义音频目录、阈值和输出文件： + +```bash +python scripts/openwakeword_ax630c.py \ + --audio-dir audio/openwakeword \ + --threshold 0.5 \ + --output outputs/openwakeword_ax630c.json +``` + +## C++ + +### 编译环境 + +AX650 和 AX630C 共用 Arm GNU 9.2 AArch64 交叉编译器： + + + +```bash +mkdir -p cpp/toolchains +wget -P cpp/toolchains \ + https://developer.arm.com/-/media/Files/downloads/gnu-a/9.2-2019.12/binrel/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu.tar.xz +tar -xf cpp/toolchains/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu.tar.xz \ + -C cpp/toolchains +``` + +BSP SDK 下载地址： + +| 平台 | BSP SDK | +| --- | --- | +| AX650 | | +| AX630C | | + +下载并整理到 `cpp/toolchains/`： + +```bash +bash cpp/download_bsp.sh all +``` + +### 交叉编译 + +```bash +export TOOLCHAIN_ROOT="cpp/toolchains/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu" + +bash cpp/build_ax650.sh +bash cpp/build_ax630c.sh +``` + +编译结果： + +```text +cpp/bin/openwakeword_ax650 +cpp/bin/openwakeword_ax630c +``` + +### 板端批量测试 + +AX650： + +```bash +THRESHOLD=0.5 bash cpp/run_batch_openwakeword_ax650.sh +``` + +AX630C： + +```bash +THRESHOLD=0.5 bash cpp/run_batch_openwakeword_ax630c.sh +``` + +结果分别写入 `outputs/cpp_ax650_batch/` 和 `outputs/cpp_ax630c_batch/`。 + +详细 C++ 说明见 `cpp/README.md`。 + +## 参考 + +- [openWakeWord](https://github.com/dscripka/openWakeWord) +- [Pulsar2](https://pulsar2-docs.readthedocs.io/en/latest/pulsar2/introduction.html) diff --git a/audio/openwakeword/alexa_test.wav b/audio/openwakeword/alexa_test.wav new file mode 100644 index 0000000000000000000000000000000000000000..637b9cd3e7869f7d1af66ddf7753518781534ff7 Binary files /dev/null and b/audio/openwakeword/alexa_test.wav differ diff --git a/audio/openwakeword/hey_jane.wav b/audio/openwakeword/hey_jane.wav new file mode 100644 index 0000000000000000000000000000000000000000..4daf9925085f507ee9a098107d4bcecd291f449d Binary files /dev/null and b/audio/openwakeword/hey_jane.wav differ diff --git a/audio/openwakeword/hey_mycroft_test.wav b/audio/openwakeword/hey_mycroft_test.wav new file mode 100644 index 0000000000000000000000000000000000000000..8ab6f213340aa91a5c5077688eddc5ce0650ffbb Binary files /dev/null and b/audio/openwakeword/hey_mycroft_test.wav differ diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..57665314598ef25fbf231b1ec27d544e78a49953 --- /dev/null +++ b/config.json @@ -0,0 +1,9 @@ +{ + "model_type": "keyword-spotting", + "source": "openWakeWord", + "target_hardware": "AX630C/AX650", + "runtime": "axengine", + "sample_rate": 16000, + "default_threshold": 0.5, + "mel_backend": "numpy" +} diff --git a/config/openwakeword_mel_weights.bin b/config/openwakeword_mel_weights.bin new file mode 100644 index 0000000000000000000000000000000000000000..4aa1e55d22c6223b9e76f1abddc2e5338b64e731 --- /dev/null +++ b/config/openwakeword_mel_weights.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5d3a5bafe0def6c1efb4874d724c272d78edadada6652ef885059db2ba147a7 +size 1085608 diff --git a/config/openwakeword_mel_weights.npz b/config/openwakeword_mel_weights.npz new file mode 100644 index 0000000000000000000000000000000000000000..3b5fc1833ceee6ccb2acc303d2c08c54adc39164 --- /dev/null +++ b/config/openwakeword_mel_weights.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3de443a7383dc51a4bf8b1d2dc9bbc767f7e33e17af617d993700a5f9947ed6 +size 1086554 diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..60283044dcd89d433bd81fda24ffb83dd9287e62 --- /dev/null +++ b/cpp/CMakeLists.txt @@ -0,0 +1,57 @@ +cmake_minimum_required(VERSION 3.13) +project(openwakeword_axera_cpp LANGUAGES C CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_BUILD_TYPE Release CACHE STRING "") + +set(AXERA_TARGET "AX650" CACHE STRING "AXERA target: AX650 or AX630C") +set_property(CACHE AXERA_TARGET PROPERTY STRINGS AX650 AX630C) +string(TOUPPER "${AXERA_TARGET}" AXERA_TARGET) +if(AXERA_TARGET STREQUAL "AX650") + set(KWS_EXECUTABLE openwakeword_ax650) + set(DEFAULT_BSP_MSP_DIR + "${CMAKE_SOURCE_DIR}/toolchains/ax650n_bsp_sdk/msp/out") +elseif(AXERA_TARGET STREQUAL "AX630C") + set(KWS_EXECUTABLE openwakeword_ax630c) + set(DEFAULT_BSP_MSP_DIR + "${CMAKE_SOURCE_DIR}/toolchains/ax620e_bsp_sdk/msp/out/arm64_glibc") +else() + message(FATAL_ERROR "AXERA_TARGET must be AX650 or AX630C") +endif() + +set(BSP_MSP_DIR "${DEFAULT_BSP_MSP_DIR}" CACHE PATH + "BSP directory containing include/ and lib/") +if(NOT EXISTS "${BSP_MSP_DIR}/include/ax_engine_api.h") + message(FATAL_ERROR + "${AXERA_TARGET} BSP not found: ${BSP_MSP_DIR}. " + "Set BSP_MSP_DIR to the directory containing include/ and lib/.") +endif() + +add_executable(${KWS_EXECUTABLE} + openwakeword_ax.cpp + src/engine_wrapper.cpp +) +target_include_directories(${KWS_EXECUTABLE} PRIVATE + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/src + ${BSP_MSP_DIR}/include +) +target_link_directories(${KWS_EXECUTABLE} PRIVATE ${BSP_MSP_DIR}/lib) +target_compile_definitions(${KWS_EXECUTABLE} PRIVATE + AXERA_TARGET_NAME="${AXERA_TARGET}" +) +target_compile_options(${KWS_EXECUTABLE} PRIVATE + -O3 -Wall -Wextra -Wpedantic + -fvisibility=hidden -fdata-sections -ffunction-sections +) +target_link_libraries(${KWS_EXECUTABLE} PRIVATE + ax_sys ax_engine ax_interpreter pthread dl m +) +target_link_options(${KWS_EXECUTABLE} PRIVATE + -Wl,--enable-new-dtags -Wl,--allow-shlib-undefined +) +set_target_properties(${KWS_EXECUTABLE} PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "$ORIGIN" +) diff --git a/cpp/README.md b/cpp/README.md new file mode 100644 index 0000000000000000000000000000000000000000..aa5b82711700b59cc5a222f1c08f4d3dafebb66b --- /dev/null +++ b/cpp/README.md @@ -0,0 +1,89 @@ +# openWakeWord C++ 推理 + +本目录提供 AX650 和 AX630C C++ 推理程序。WAV、STFT、mel 和缓存更新在 CPU +端完成，embedding 与 classifier 使用对应平台的 `axmodel`。 + +## 编译环境 + +AX650 与 AX630C 共用 Arm GNU 9.2 AArch64 交叉编译器： + +```bash +mkdir -p cpp/toolchains +wget -P cpp/toolchains \ + https://developer.arm.com/-/media/Files/downloads/gnu-a/9.2-2019.12/binrel/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu.tar.xz +tar -xf cpp/toolchains/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu.tar.xz \ + -C cpp/toolchains +``` + +下载 BSP： + +```bash +bash cpp/download_bsp.sh all +``` + +| 平台 | BSP 压缩包 | 编译目录 | +| --- | --- | --- | +| AX650 | `msp_50_3.10.2.zip` | `cpp/toolchains/ax650n_bsp_sdk/msp/out` | +| AX630C | `msp_20e_3.0.0.zip` | `cpp/toolchains/ax620e_bsp_sdk/msp/out/arm64_glibc` | + +## 交叉编译 + +```bash +export TOOLCHAIN_ROOT="cpp/toolchains/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu" + +bash cpp/build_ax650.sh +bash cpp/build_ax630c.sh +``` + +生成： + +```text +cpp/bin/openwakeword_ax650 +cpp/bin/openwakeword_ax630c +``` + +如 BSP 放在其他位置，可设置 `BSP_MSP_DIR`。AX650 应指向包含 `include/` 和 +`lib/` 的 `msp/out`；AX630C 应指向 `msp/out/arm64_glibc`。 + +## 板端运行 + +AX650 单条推理： + +```bash +AUDIO=audio/openwakeword/alexa_test.wav \ +bash cpp/run_openwakeword_ax650.sh +``` + +AX630C 单条推理： + +```bash +AUDIO=audio/openwakeword/alexa_test.wav \ +bash cpp/run_openwakeword_ax630c.sh +``` + +批量测试： + +```bash +THRESHOLD=0.5 bash cpp/run_batch_openwakeword_ax650.sh +THRESHOLD=0.5 bash cpp/run_batch_openwakeword_ax630c.sh +``` + +模型目录必须与平台对应： + +```text +models/650/ +models/630C/ +``` + +运行脚本会检查 embedding 和 6 个 classifier 模型，避免误用其他平台的 axmodel。 +输入音频必须为 16 kHz、单声道、16-bit PCM WAV。 + +单条推理会输出 classifier 分数和 `WAKEUP`，并只输出一个 `rtf`。`rtf` 包含 +CPU mel 和 NPU 推理，不包含一次性模型加载与初始化。 + +阈值可通过环境变量调整： + +```bash +THRESHOLD=0.6 AUDIO=audio/openwakeword/hey_mycroft_test.wav \ +bash cpp/run_openwakeword_ax630c.sh +``` diff --git a/cpp/bin/openwakeword_ax630c b/cpp/bin/openwakeword_ax630c new file mode 100644 index 0000000000000000000000000000000000000000..0e028c6ed2d19f534149dc5c4f4d567c87f81956 Binary files /dev/null and b/cpp/bin/openwakeword_ax630c differ diff --git a/cpp/bin/openwakeword_ax650 b/cpp/bin/openwakeword_ax650 new file mode 100644 index 0000000000000000000000000000000000000000..1e6810b01f4820b2e673b4cc0d6ab1ca1b3bc7d3 Binary files /dev/null and b/cpp/bin/openwakeword_ax650 differ diff --git a/cpp/build_ax630c.sh b/cpp/build_ax630c.sh new file mode 100644 index 0000000000000000000000000000000000000000..dab42b465e80242e31063cdf8abb1098664ef762 --- /dev/null +++ b/cpp/build_ax630c.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CPP_DIR="${ROOT}/cpp" +TOOLCHAIN_ROOT="${TOOLCHAIN_ROOT:-${CPP_DIR}/toolchains/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu}" +BSP_MSP_DIR="${BSP_MSP_DIR:-${CPP_DIR}/toolchains/ax620e_bsp_sdk/msp/out/arm64_glibc}" +BUILD_DIR="${CPP_DIR}/build/ax630c" + +if [[ ! -x "${TOOLCHAIN_ROOT}/bin/aarch64-none-linux-gnu-g++" ]]; then + echo "ERROR: AArch64 toolchain not found: ${TOOLCHAIN_ROOT}" >&2 + exit 2 +fi +if [[ ! -f "${BSP_MSP_DIR}/include/ax_engine_api.h" ]]; then + echo "ERROR: AX630C BSP arm64_glibc directory not found: ${BSP_MSP_DIR}" >&2 + exit 2 +fi + +mkdir -p "${BUILD_DIR}" "${CPP_DIR}/bin" +cmake -S "${CPP_DIR}" -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_SYSTEM_PROCESSOR=aarch64 \ + -DCMAKE_C_COMPILER="${TOOLCHAIN_ROOT}/bin/aarch64-none-linux-gnu-gcc" \ + -DCMAKE_CXX_COMPILER="${TOOLCHAIN_ROOT}/bin/aarch64-none-linux-gnu-g++" \ + -DAXERA_TARGET=AX630C \ + -DBSP_MSP_DIR="${BSP_MSP_DIR}" +cmake --build "${BUILD_DIR}" -j"$(nproc)" +cp "${BUILD_DIR}/openwakeword_ax630c" "${CPP_DIR}/bin/" +"${TOOLCHAIN_ROOT}/bin/aarch64-none-linux-gnu-strip" --strip-unneeded \ + "${CPP_DIR}/bin/openwakeword_ax630c" +echo "Built ${CPP_DIR}/bin/openwakeword_ax630c" diff --git a/cpp/build_ax650.sh b/cpp/build_ax650.sh new file mode 100644 index 0000000000000000000000000000000000000000..400ce009cb432af7de1fe5a2af2751108fcc7c3a --- /dev/null +++ b/cpp/build_ax650.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CPP_DIR="${ROOT}/cpp" +TOOLCHAIN_ROOT="${TOOLCHAIN_ROOT:-${CPP_DIR}/toolchains/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu}" +BSP_MSP_DIR="${BSP_MSP_DIR:-${CPP_DIR}/toolchains/ax650n_bsp_sdk/msp/out}" +BUILD_DIR="${CPP_DIR}/build/ax650" + +if [[ ! -x "${TOOLCHAIN_ROOT}/bin/aarch64-none-linux-gnu-g++" ]]; then + echo "ERROR: AArch64 toolchain not found: ${TOOLCHAIN_ROOT}" >&2 + exit 2 +fi +if [[ ! -f "${BSP_MSP_DIR}/include/ax_engine_api.h" ]]; then + echo "ERROR: AX650 BSP msp/out not found: ${BSP_MSP_DIR}" >&2 + exit 2 +fi + +mkdir -p "${BUILD_DIR}" "${CPP_DIR}/bin" +cmake -S "${CPP_DIR}" -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_SYSTEM_PROCESSOR=aarch64 \ + -DCMAKE_C_COMPILER="${TOOLCHAIN_ROOT}/bin/aarch64-none-linux-gnu-gcc" \ + -DCMAKE_CXX_COMPILER="${TOOLCHAIN_ROOT}/bin/aarch64-none-linux-gnu-g++" \ + -DAXERA_TARGET=AX650 \ + -DBSP_MSP_DIR="${BSP_MSP_DIR}" +cmake --build "${BUILD_DIR}" -j"$(nproc)" +cp "${BUILD_DIR}/openwakeword_ax650" "${CPP_DIR}/bin/" +"${TOOLCHAIN_ROOT}/bin/aarch64-none-linux-gnu-strip" --strip-unneeded \ + "${CPP_DIR}/bin/openwakeword_ax650" +echo "Built ${CPP_DIR}/bin/openwakeword_ax650" diff --git a/cpp/download_bsp.sh b/cpp/download_bsp.sh new file mode 100644 index 0000000000000000000000000000000000000000..4e43bee92a8aa37bf3e7e0d9bc6c6f50d4eeea74 --- /dev/null +++ b/cpp/download_bsp.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +CPP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOOLCHAINS_DIR="${CPP_DIR}/toolchains" +PLATFORM="${1:-all}" + +download_sdk() { + local directory="$1" + local archive_name="$2" + local extracted_name="$3" + local url="$4" + + if [[ -d "${TOOLCHAINS_DIR}/${directory}" ]]; then + echo "Found ${TOOLCHAINS_DIR}/${directory}" + return + fi + mkdir -p "${TOOLCHAINS_DIR}" + wget -c "${url}" -O "${TOOLCHAINS_DIR}/${archive_name}" + unzip -q "${TOOLCHAINS_DIR}/${archive_name}" -d "${TOOLCHAINS_DIR}" + mv "${TOOLCHAINS_DIR}/${extracted_name}" "${TOOLCHAINS_DIR}/${directory}" + rm -f "${TOOLCHAINS_DIR}/${archive_name}" + echo "Downloaded ${TOOLCHAINS_DIR}/${directory}" +} + +case "${PLATFORM}" in + 650|AX650|ax650) + download_sdk \ + ax650n_bsp_sdk msp_50_3.10.2.zip msp_50_3.10.2 \ + https://github.com/ZHEQIUSHUI/assets/releases/download/ax_3.6.2/msp_50_3.10.2.zip + ;; + 630C|AX630C|ax630c) + download_sdk \ + ax620e_bsp_sdk msp_20e_3.0.0.zip msp_20e_3.0.0 \ + https://github.com/ZHEQIUSHUI/assets/releases/download/ax_3.6.2/msp_20e_3.0.0.zip + ;; + all) + bash "${BASH_SOURCE[0]}" 650 + bash "${BASH_SOURCE[0]}" 630C + ;; + *) + echo "Usage: bash cpp/download_bsp.sh [650|630C|all]" >&2 + exit 2 + ;; +esac diff --git a/cpp/openwakeword_ax.cpp b/cpp/openwakeword_ax.cpp new file mode 100644 index 0000000000000000000000000000000000000000..61ac6ee0b24e3dd2f68ca240dc0171cbac248d5f --- /dev/null +++ b/cpp/openwakeword_ax.cpp @@ -0,0 +1,379 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ax_engine_api.h" +#include "ax_sys_api.h" +#include "src/engine_wrapper.hpp" +#include "src/wav_reader.hpp" + +#ifndef AXERA_TARGET_NAME +#define AXERA_TARGET_NAME "AXERA" +#endif + +namespace { +constexpr int kSampleRate = 16000; +constexpr int kChunkSamples = 1280; +constexpr int kHistorySamples = 480; +constexpr int kFftSize = 512; +constexpr int kSpectrumBins = 257; +constexpr int kMelBins = 32; +constexpr int kMelFrames = 8; +constexpr int kEmbeddingFrames = 76; +constexpr int kEmbeddingSize = 96; +constexpr int kFeatureFrames = 34; +using Clock = std::chrono::steady_clock; + +double ElapsedSeconds(Clock::time_point begin, Clock::time_point end) { + return std::chrono::duration(end - begin).count(); +} + +struct Args { + std::string models_dir = "models"; + std::string weights = "config/openwakeword_mel_weights.bin"; + std::string audio = "audio/openwakeword/alexa_test.wav"; + float threshold = 0.5f; +}; + +void Usage(const char *program) { + std::printf( + "Usage: %s [--models-dir DIR] [--mel-weights FILE] [--audio WAV] " + "[--threshold VALUE]\n", + program); +} + +Args ParseArgs(int argc, char **argv) { + Args args; + for (int i = 1; i < argc; ++i) { + const std::string key = argv[i]; + auto value = [&]() -> std::string { + if (++i >= argc) throw std::runtime_error("Missing value for " + key); + return argv[i]; + }; + if (key == "--models-dir") { + args.models_dir = value(); + } else if (key == "--mel-weights") { + args.weights = value(); + } else if (key == "--audio") { + args.audio = value(); + } else if (key == "--threshold") { + args.threshold = std::stof(value()); + } else if (key == "-h" || key == "--help") { + Usage(argv[0]); + std::exit(0); + } else { + throw std::runtime_error("Unknown argument: " + key); + } + } + return args; +} + +std::string Join(const std::string &left, const std::string &right) { + return left.empty() || left.back() == '/' ? left + right + : left + "/" + right; +} + +class AxRuntime { + public: + AxRuntime() { + if (AX_SYS_Init() != 0) throw std::runtime_error("AX_SYS_Init failed"); + sys_initialized_ = true; + AX_ENGINE_NPU_ATTR_T attr{}; + if (AX_ENGINE_Init(&attr) != 0) { + AX_SYS_Deinit(); + sys_initialized_ = false; + throw std::runtime_error("AX_ENGINE_Init failed"); + } + engine_initialized_ = true; + } + ~AxRuntime() { + if (engine_initialized_) AX_ENGINE_Deinit(); + if (sys_initialized_) AX_SYS_Deinit(); + } + + private: + bool sys_initialized_ = false; + bool engine_initialized_ = false; +}; + +template +T ReadScalar(std::istream &input) { + T value{}; + input.read(reinterpret_cast(&value), sizeof(value)); + if (!input) throw std::runtime_error("Truncated mel weight file"); + return value; +} + +struct MelWeights { + std::vector real; + std::vector imag; + std::vector mel; + float floor = 0.0f; + + static MelWeights Load(const std::string &path) { + std::ifstream input(path, std::ios::binary); + if (!input) throw std::runtime_error("Cannot open mel weights: " + path); + char magic[8]{}; + input.read(magic, 8); + if (std::memcmp(magic, "OWWMEL1", 7) != 0 || + ReadScalar(input) != 1) { + throw std::runtime_error("Invalid openWakeWord mel weight file"); + } + const uint32_t real_rows = ReadScalar(input); + const uint32_t real_cols = ReadScalar(input); + const uint32_t imag_rows = ReadScalar(input); + const uint32_t imag_cols = ReadScalar(input); + const uint32_t mel_rows = ReadScalar(input); + const uint32_t mel_cols = ReadScalar(input); + MelWeights result; + result.floor = ReadScalar(input); + if (real_rows != kSpectrumBins || real_cols != kFftSize || + imag_rows != kSpectrumBins || imag_cols != kFftSize || + mel_rows != kSpectrumBins || mel_cols != kMelBins) { + throw std::runtime_error("Unexpected openWakeWord mel weight shapes"); + } + result.real.resize(static_cast(real_rows) * real_cols); + result.imag.resize(static_cast(imag_rows) * imag_cols); + result.mel.resize(static_cast(mel_rows) * mel_cols); + input.read(reinterpret_cast(result.real.data()), + result.real.size() * sizeof(float)); + input.read(reinterpret_cast(result.imag.data()), + result.imag.size() * sizeof(float)); + input.read(reinterpret_cast(result.mel.data()), + result.mel.size() * sizeof(float)); + if (!input) throw std::runtime_error("Truncated openWakeWord mel weights"); + return result; + } +}; + +std::array ComputeMel( + const std::array &samples, + const MelWeights &weights) { + std::array result{}; + std::array power{}; + float max_db = -std::numeric_limits::infinity(); + for (int frame = 0; frame < kMelFrames; ++frame) { + const float *frame_samples = samples.data() + frame * 160; + for (int frequency = 0; frequency < kSpectrumBins; ++frequency) { + const float *real = weights.real.data() + frequency * kFftSize; + const float *imag = weights.imag.data() + frequency * kFftSize; + float real_sum = 0.0f; + float imag_sum = 0.0f; + for (int n = 0; n < kFftSize; ++n) { + real_sum += frame_samples[n] * real[n]; + imag_sum += frame_samples[n] * imag[n]; + } + power[frequency] = real_sum * real_sum + imag_sum * imag_sum; + } + for (int bin = 0; bin < kMelBins; ++bin) { + float value = 0.0f; + for (int frequency = 0; frequency < kSpectrumBins; ++frequency) { + value += power[frequency] * + weights.mel[frequency * kMelBins + bin]; + } + value = std::max(value, weights.floor); + const float db = std::log(value) * 10.0f / 2.3025851249694824f; + result[frame * kMelBins + bin] = db; + max_db = std::max(max_db, db); + } + } + const float minimum = max_db - 80.0f; + for (float &value : result) { + value = std::max(value, minimum) / 10.0f + 2.0f; + } + return result; +} + +struct Classifier { + std::string name; + int frames; + EngineWrapper engine; + std::vector maximum; +}; + +void Run(const Args &args) { + const PcmWav wav = ReadPcmWav(args.audio); + if (wav.sample_rate != kSampleRate) { + throw std::runtime_error("Input WAV must use 16 kHz sample rate"); + } + const double audio_seconds = + static_cast(wav.samples.size()) / kSampleRate; + if (audio_seconds <= 0.0) { + throw std::runtime_error("Input WAV contains no samples"); + } + const MelWeights weights = MelWeights::Load(args.weights); + const auto model_load_begin = Clock::now(); + AxRuntime runtime; + + EngineWrapper embedding; + if (embedding.Init(Join(args.models_dir, + "openwakeword__embedding_model.axmodel")) != 0) { + throw std::runtime_error("Failed to load embedding model"); + } + const std::array, 6> definitions{{ + {"alexa_v0.1", 16}, + {"hey_jarvis_v0.1", 16}, + {"hey_mycroft_v0.1", 16}, + {"hey_rhasspy_v0.1", 16}, + {"timer_v0.1", 34}, + {"weather_v0.1", 22}, + }}; + std::array classifiers; + for (std::size_t classifier_index = 0; + classifier_index < definitions.size(); ++classifier_index) { + const auto &definition = definitions[classifier_index]; + Classifier &classifier = classifiers[classifier_index]; + classifier.name = definition.first; + classifier.frames = definition.second; + const std::string model = + Join(args.models_dir, "openwakeword__" + classifier.name + ".axmodel"); + if (classifier.engine.Init(model) != 0) { + throw std::runtime_error("Failed to load classifier: " + classifier.name); + } + const int output_bytes = classifier.engine.GetOutputSizeByName( + classifier.engine.OutputName(0)); + if (output_bytes <= 0 || output_bytes % sizeof(float) != 0) { + throw std::runtime_error("Unexpected classifier output: " + + classifier.name); + } + classifier.maximum.assign(output_bytes / sizeof(float), + -std::numeric_limits::infinity()); + } + const double model_load_seconds = + ElapsedSeconds(model_load_begin, Clock::now()); + + std::vector padded = wav.samples; + const std::size_t remainder = padded.size() % kChunkSamples; + if (remainder != 0) padded.resize(padded.size() + kChunkSamples - remainder); + std::array history{}; + std::array mel_buffer{}; + mel_buffer.fill(1.0f); + std::array feature_buffer{}; + int chunks = 0; + double feature_seconds = 0.0; + double npu_seconds = 0.0; + const auto inference_begin = Clock::now(); + + for (std::size_t start = 0; start < padded.size(); start += kChunkSamples) { + std::array mel_input{}; + for (int i = 0; i < kHistorySamples; ++i) mel_input[i] = history[i]; + for (int i = 0; i < kChunkSamples; ++i) { + mel_input[kHistorySamples + i] = padded[start + i]; + } + for (int i = 0; i < kHistorySamples; ++i) { + history[i] = padded[start + kChunkSamples - kHistorySamples + i]; + } + const auto feature_begin = Clock::now(); + const auto mel = ComputeMel(mel_input, weights); + feature_seconds += ElapsedSeconds(feature_begin, Clock::now()); + std::memmove(mel_buffer.data(), mel_buffer.data() + kMelFrames * kMelBins, + (kEmbeddingFrames - kMelFrames) * kMelBins * sizeof(float)); + std::memcpy(mel_buffer.data() + + (kEmbeddingFrames - kMelFrames) * kMelBins, + mel.data(), mel.size() * sizeof(float)); + + const std::string &embedding_input = embedding.InputName(0); + if (embedding.SetInputByName(embedding_input, mel_buffer.data(), + mel_buffer.size() * sizeof(float)) != 0) { + throw std::runtime_error("Failed to set embedding input"); + } + const auto embedding_begin = Clock::now(); + const int embedding_ret = embedding.RunSync(); + npu_seconds += ElapsedSeconds(embedding_begin, Clock::now()); + if (embedding_ret != 0) { + throw std::runtime_error("Embedding inference failed"); + } + std::array feature{}; + if (embedding.GetOutputByName(embedding.OutputName(0), feature.data(), + feature.size() * sizeof(float)) != 0) { + throw std::runtime_error("Failed to read embedding output"); + } + std::memmove(feature_buffer.data(), feature_buffer.data() + kEmbeddingSize, + (kFeatureFrames - 1) * kEmbeddingSize * sizeof(float)); + std::memcpy(feature_buffer.data() + + (kFeatureFrames - 1) * kEmbeddingSize, + feature.data(), feature.size() * sizeof(float)); + + for (Classifier &classifier : classifiers) { + const float *input = + feature_buffer.data() + + (kFeatureFrames - classifier.frames) * kEmbeddingSize; + const std::string &input_name = classifier.engine.InputName(0); + if (classifier.engine.SetInputByName( + input_name, input, + classifier.frames * kEmbeddingSize * sizeof(float)) != 0) { + throw std::runtime_error("Failed to set classifier input: " + + classifier.name); + } + const auto classifier_begin = Clock::now(); + const int classifier_ret = classifier.engine.RunSync(); + npu_seconds += ElapsedSeconds(classifier_begin, Clock::now()); + if (classifier_ret != 0) { + throw std::runtime_error("Classifier inference failed: " + + classifier.name); + } + std::vector output(classifier.maximum.size()); + if (classifier.engine.GetOutputByName( + classifier.engine.OutputName(0), output.data(), + output.size() * sizeof(float)) != 0) { + throw std::runtime_error("Failed to read classifier output"); + } + for (std::size_t i = 0; i < output.size(); ++i) { + classifier.maximum[i] = std::max(classifier.maximum[i], output[i]); + } + } + ++chunks; + } + const double inference_seconds = + ElapsedSeconds(inference_begin, Clock::now()); + const double rtf = inference_seconds / audio_seconds; + + std::printf("\nopenWakeWord C++ inference complete\n"); + std::printf("target: %s\naudio: %s\nchunks: %d\nthreshold: %.3f\n", + AXERA_TARGET_NAME, args.audio.c_str(), chunks, args.threshold); + bool detected = false; + for (const Classifier &classifier : classifiers) { + std::printf("%-22s", classifier.name.c_str()); + float score = -std::numeric_limits::infinity(); + for (std::size_t i = 0; i < classifier.maximum.size(); ++i) { + const float value = classifier.maximum[i]; + std::printf(" %.6f", value); + if (classifier.name != "timer_v0.1" || i != 0) { + score = std::max(score, value); + } + } + if (score >= args.threshold) { + std::printf(" WAKEUP"); + detected = true; + } + std::printf("\n"); + } + std::printf("detected: %s\n", detected ? "true" : "false"); + std::printf("audio_seconds: %.6f\n", audio_seconds); + std::printf("feature_seconds: %.6f\n", feature_seconds); + std::printf("npu_seconds: %.6f\n", npu_seconds); + std::printf("model_load_seconds: %.6f\n", model_load_seconds); + std::printf("inference_seconds: %.6f\n", inference_seconds); + std::printf("rtf: %.6f\n", rtf); +} +} // namespace + +int main(int argc, char **argv) { + try { + const Args args = ParseArgs(argc, argv); + Run(args); + return 0; + } catch (const std::exception &error) { + std::fprintf(stderr, "ERROR: %s\n", error.what()); + return 1; + } +} diff --git a/cpp/run_batch_openwakeword_ax.sh b/cpp/run_batch_openwakeword_ax.sh new file mode 100644 index 0000000000000000000000000000000000000000..fb78efcfdba83c40b21b9a6df2be81572ea4409a --- /dev/null +++ b/cpp/run_batch_openwakeword_ax.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TARGET="${AXERA_TARGET:-AX650}" +AUDIO_DIR="${AUDIO_DIR:-${ROOT}/audio/openwakeword}" +THRESHOLD="${THRESHOLD:-0.5}" + +case "${TARGET}" in + AX650) + TARGET_SLUG="ax650" + BINARY="${ROOT}/cpp/bin/openwakeword_ax650" + MODELS_DIR="${MODELS_DIR:-${ROOT}/models/650}" + BSP_LIB_DIR="${AXERA_BSP_LIB_DIR:-${AX650_BSP_LIB_DIR:-${ROOT}/cpp/toolchains/ax650n_bsp_sdk/msp/out/lib}}" + BUILD_SCRIPT="cpp/build_ax650.sh" + ;; + AX630C) + TARGET_SLUG="ax630c" + BINARY="${ROOT}/cpp/bin/openwakeword_ax630c" + MODELS_DIR="${MODELS_DIR:-${ROOT}/models/630C}" + BSP_LIB_DIR="${AXERA_BSP_LIB_DIR:-${AX630C_BSP_LIB_DIR:-${ROOT}/cpp/toolchains/ax620e_bsp_sdk/msp/out/arm64_glibc/lib}}" + BUILD_SCRIPT="cpp/build_ax630c.sh" + ;; + *) + echo "ERROR: AXERA_TARGET must be AX650 or AX630C" >&2 + exit 2 + ;; +esac + +OUTPUT_DIR="${OUTPUT_DIR:-${ROOT}/outputs/cpp_${TARGET_SLUG}_batch}" +if [[ ! -x "${BINARY}" ]]; then + echo "ERROR: ${BINARY} is missing; run bash ${BUILD_SCRIPT} first" >&2 + exit 2 +fi +if [[ ! -d "${AUDIO_DIR}" ]]; then + echo "ERROR: audio directory does not exist: ${AUDIO_DIR}" >&2 + exit 2 +fi +for name in embedding_model alexa_v0.1 hey_jarvis_v0.1 hey_mycroft_v0.1 \ + hey_rhasspy_v0.1 timer_v0.1 weather_v0.1; do + model="${MODELS_DIR}/openwakeword__${name}.axmodel" + if [[ ! -f "${model}" ]]; then + echo "ERROR: ${TARGET} model is missing: ${model}" >&2 + exit 2 + fi +done + +mapfile -d '' AUDIO_FILES < <( + find "${AUDIO_DIR}" -maxdepth 1 -type f -iname '*.wav' -print0 | sort -z +) +if [[ "${#AUDIO_FILES[@]}" -eq 0 ]]; then + echo "ERROR: no WAV files found under ${AUDIO_DIR}" >&2 + exit 2 +fi + +mkdir -p "${OUTPUT_DIR}/logs" +RESULTS="${OUTPUT_DIR}/results.tsv" +SUMMARY="${OUTPUT_DIR}/summary.txt" +printf 'file\taudio_seconds\tfeature_seconds\tnpu_seconds\tinference_seconds\trtf\tmodel_load_seconds\tdetections\n' > "${RESULTS}" + +export LD_LIBRARY_PATH="${BSP_LIB_DIR}:${LD_LIBRARY_PATH:-}" +for audio in "${AUDIO_FILES[@]}"; do + name="$(basename "${audio}")" + stem="${name%.*}" + log="${OUTPUT_DIR}/logs/${stem}.log" + echo "[RUN] target=${TARGET} ${name} threshold=${THRESHOLD}" + "${BINARY}" \ + --models-dir "${MODELS_DIR}" \ + --mel-weights "${ROOT}/config/openwakeword_mel_weights.bin" \ + --audio "${audio}" \ + --threshold "${THRESHOLD}" > "${log}" 2>&1 + cat "${log}" + + field() { + awk -F': ' -v key="$1" '$1 == key {print $2; exit}' "${log}" + } + audio_seconds="$(field audio_seconds)" + feature_seconds="$(field feature_seconds)" + npu_seconds="$(field npu_seconds)" + inference_seconds="$(field inference_seconds)" + rtf="$(field rtf)" + model_load_seconds="$(field model_load_seconds)" + detections="$(awk '$NF == "WAKEUP" {print $1}' "${log}" | paste -sd ',' -)" + detections="${detections:-none}" + + for value in "${audio_seconds}" "${feature_seconds}" "${npu_seconds}" \ + "${inference_seconds}" "${rtf}" "${model_load_seconds}"; do + if [[ -z "${value}" ]]; then + echo "ERROR: timing field missing from ${log}" >&2 + exit 1 + fi + done + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${name}" "${audio_seconds}" "${feature_seconds}" "${npu_seconds}" \ + "${inference_seconds}" "${rtf}" "${model_load_seconds}" "${detections}" \ + >> "${RESULTS}" +done + +awk -F'\t' ' + NR > 1 { + files += 1 + audio += $2 + feature += $3 + npu += $4 + inference += $5 + model_load += $7 + } + END { + printf "files: %d\n", files + printf "total_audio_seconds: %.6f\n", audio + printf "total_feature_seconds: %.6f\n", feature + printf "total_npu_seconds: %.6f\n", npu + printf "total_inference_seconds: %.6f\n", inference + printf "total_model_load_seconds: %.6f\n", model_load + printf "rtf: %.6f\n", inference / audio + } +' "${RESULTS}" > "${SUMMARY}" + +echo "[SUMMARY]" +cat "${SUMMARY}" +echo "results: ${RESULTS}" +echo "logs: ${OUTPUT_DIR}/logs" diff --git a/cpp/run_batch_openwakeword_ax630c.sh b/cpp/run_batch_openwakeword_ax630c.sh new file mode 100644 index 0000000000000000000000000000000000000000..c6e114b1f18639ce80ab843637c56c3ae06b4073 --- /dev/null +++ b/cpp/run_batch_openwakeword_ax630c.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +AXERA_TARGET=AX630C exec bash "${ROOT}/cpp/run_batch_openwakeword_ax.sh" diff --git a/cpp/run_batch_openwakeword_ax650.sh b/cpp/run_batch_openwakeword_ax650.sh new file mode 100644 index 0000000000000000000000000000000000000000..02f33b4d9371cb838ecb7daa48628f641117a40b --- /dev/null +++ b/cpp/run_batch_openwakeword_ax650.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +AXERA_TARGET=AX650 exec bash "${ROOT}/cpp/run_batch_openwakeword_ax.sh" diff --git a/cpp/run_openwakeword_ax.sh b/cpp/run_openwakeword_ax.sh new file mode 100644 index 0000000000000000000000000000000000000000..eb88fa1ec6f56751b479418bc678ec9b59233e5c --- /dev/null +++ b/cpp/run_openwakeword_ax.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TARGET="${AXERA_TARGET:-AX650}" +AUDIO="${AUDIO:-${ROOT}/audio/openwakeword/alexa_test.wav}" +THRESHOLD="${THRESHOLD:-0.5}" + +case "${TARGET}" in + AX650) + BINARY="${ROOT}/cpp/bin/openwakeword_ax650" + MODELS_DIR="${MODELS_DIR:-${ROOT}/models/650}" + BSP_LIB_DIR="${AXERA_BSP_LIB_DIR:-${AX650_BSP_LIB_DIR:-${ROOT}/cpp/toolchains/ax650n_bsp_sdk/msp/out/lib}}" + BUILD_SCRIPT="cpp/build_ax650.sh" + ;; + AX630C) + BINARY="${ROOT}/cpp/bin/openwakeword_ax630c" + MODELS_DIR="${MODELS_DIR:-${ROOT}/models/630C}" + BSP_LIB_DIR="${AXERA_BSP_LIB_DIR:-${AX630C_BSP_LIB_DIR:-${ROOT}/cpp/toolchains/ax620e_bsp_sdk/msp/out/arm64_glibc/lib}}" + BUILD_SCRIPT="cpp/build_ax630c.sh" + ;; + *) + echo "ERROR: AXERA_TARGET must be AX650 or AX630C" >&2 + exit 2 + ;; +esac + +if [[ ! -x "${BINARY}" ]]; then + echo "ERROR: ${BINARY} is missing; run bash ${BUILD_SCRIPT} first" >&2 + exit 2 +fi +for name in embedding_model alexa_v0.1 hey_jarvis_v0.1 hey_mycroft_v0.1 \ + hey_rhasspy_v0.1 timer_v0.1 weather_v0.1; do + model="${MODELS_DIR}/openwakeword__${name}.axmodel" + if [[ ! -f "${model}" ]]; then + echo "ERROR: ${TARGET} model is missing: ${model}" >&2 + exit 2 + fi +done + +export LD_LIBRARY_PATH="${BSP_LIB_DIR}:${LD_LIBRARY_PATH:-}" +exec "${BINARY}" \ + --models-dir "${MODELS_DIR}" \ + --mel-weights "${ROOT}/config/openwakeword_mel_weights.bin" \ + --audio "${AUDIO}" \ + --threshold "${THRESHOLD}" diff --git a/cpp/run_openwakeword_ax630c.sh b/cpp/run_openwakeword_ax630c.sh new file mode 100644 index 0000000000000000000000000000000000000000..5405af2edb38ecc3efef07903f3dcb26e7f674a5 --- /dev/null +++ b/cpp/run_openwakeword_ax630c.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +AXERA_TARGET=AX630C exec bash "${ROOT}/cpp/run_openwakeword_ax.sh" diff --git a/cpp/run_openwakeword_ax650.sh b/cpp/run_openwakeword_ax650.sh new file mode 100644 index 0000000000000000000000000000000000000000..df2b424f15a434c67c63d7c07bb62d2c57c115c0 --- /dev/null +++ b/cpp/run_openwakeword_ax650.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +AXERA_TARGET=AX650 exec bash "${ROOT}/cpp/run_openwakeword_ax.sh" diff --git a/cpp/src/engine_wrapper.cpp b/cpp/src/engine_wrapper.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d84d16ccb50f03d0c07668abff32e24691d3b3ec --- /dev/null +++ b/cpp/src/engine_wrapper.cpp @@ -0,0 +1,197 @@ +#include "engine_wrapper.hpp" + +#include +#include +#include +#include + +#include "ax_sys_api.h" + +namespace { +constexpr int kAlignment = 128; + +void FreeIo(AX_ENGINE_IO_T *io) { + for (AX_U32 i = 0; io->pInputs && i < io->nInputSize; ++i) { + if (io->pInputs[i].pVirAddr) { + AX_SYS_MemFree(io->pInputs[i].phyAddr, io->pInputs[i].pVirAddr); + } + } + for (AX_U32 i = 0; io->pOutputs && i < io->nOutputSize; ++i) { + if (io->pOutputs[i].pVirAddr) { + AX_SYS_MemFree(io->pOutputs[i].phyAddr, io->pOutputs[i].pVirAddr); + } + } + delete[] io->pInputs; + delete[] io->pOutputs; + std::memset(io, 0, sizeof(*io)); +} +} // namespace + +EngineWrapper::~EngineWrapper() { Release(); } + +int EngineWrapper::Init(const std::string &path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + std::fprintf(stderr, "Failed to read model: %s\n", path.c_str()); + return -1; + } + input.seekg(0, std::ios::end); + const std::streamoff model_size = input.tellg(); + if (model_size <= 0 || model_size > static_cast(UINT32_MAX)) { + std::fprintf(stderr, "Invalid model size: %s\n", path.c_str()); + return -1; + } + input.seekg(0, std::ios::beg); + std::vector model(static_cast(model_size)); + input.read(model.data(), model_size); + if (!input) { + std::fprintf(stderr, "Failed to read model: %s\n", path.c_str()); + return -1; + } + const AX_S32 create_ret = AX_ENGINE_CreateHandle( + &handle_, model.data(), static_cast(model.size())); + if (create_ret != 0 || !handle_) { + std::fprintf(stderr, "AX_ENGINE_CreateHandle failed: %s ret=0x%x\n", + path.c_str(), static_cast(create_ret)); + return -1; + } + if (AX_ENGINE_CreateContext(handle_) != 0 || + AX_ENGINE_GetIOInfo(handle_, &info_) != 0 || !info_) { + Release(); + return -1; + } + + io_.nInputSize = info_->nInputSize; + io_.nOutputSize = info_->nOutputSize; + io_.pInputs = new AX_ENGINE_IO_BUFFER_T[io_.nInputSize]{}; + io_.pOutputs = new AX_ENGINE_IO_BUFFER_T[io_.nOutputSize]{}; + AX_S8 input_tag[] = "KWS-INPUT"; + AX_S8 output_tag[] = "KWS-OUTPUT"; + for (AX_U32 i = 0; i < io_.nInputSize; ++i) { + io_.pInputs[i].nSize = info_->pInputs[i].nSize; + if (AX_SYS_MemAlloc(&io_.pInputs[i].phyAddr, + &io_.pInputs[i].pVirAddr, io_.pInputs[i].nSize, + kAlignment, input_tag) != 0) { + Release(); + return -1; + } + const std::string name = info_->pInputs[i].pName; + inputs_[name] = static_cast(i); + input_names_.push_back(name); + } + for (AX_U32 i = 0; i < io_.nOutputSize; ++i) { + io_.pOutputs[i].nSize = info_->pOutputs[i].nSize; + if (AX_SYS_MemAlloc(&io_.pOutputs[i].phyAddr, + &io_.pOutputs[i].pVirAddr, io_.pOutputs[i].nSize, + kAlignment, output_tag) != 0) { + Release(); + return -1; + } + const std::string name = info_->pOutputs[i].pName; + outputs_[name] = static_cast(i); + output_names_.push_back(name); + } + initialized_ = true; + std::printf("Loaded %s (inputs=%u outputs=%u)\n", path.c_str(), + io_.nInputSize, io_.nOutputSize); + return 0; +} + +void EngineWrapper::Release() { + FreeIo(&io_); + if (handle_) AX_ENGINE_DestroyHandle(handle_); + initialized_ = false; + handle_ = nullptr; + info_ = nullptr; + inputs_.clear(); + outputs_.clear(); + input_names_.clear(); + output_names_.clear(); +} + +int EngineWrapper::InputIndex(const std::string &name) const { + const auto it = inputs_.find(name); + return it == inputs_.end() ? -1 : it->second; +} + +int EngineWrapper::OutputIndex(const std::string &name) const { + const auto it = outputs_.find(name); + return it == outputs_.end() ? -1 : it->second; +} + +int EngineWrapper::SetInputByName(const std::string &name, const void *data, + std::size_t size) { + const int index = InputIndex(name); + if (!initialized_ || index < 0 || !data) return -1; + const std::size_t expected = io_.pInputs[index].nSize; + if (size != 0 && size != expected) return -1; + std::memcpy(io_.pInputs[index].pVirAddr, data, expected); + return 0; +} + +int EngineWrapper::ZeroInputByName(const std::string &name) { + const int index = InputIndex(name); + if (!initialized_ || index < 0) return -1; + std::memset(io_.pInputs[index].pVirAddr, 0, io_.pInputs[index].nSize); + return 0; +} + +int EngineWrapper::RunSync() { + if (!initialized_) return -1; + for (AX_U32 i = 0; i < io_.nInputSize; ++i) { + const AX_S32 flush_ret = AX_SYS_MflushCache( + io_.pInputs[i].phyAddr, io_.pInputs[i].pVirAddr, + io_.pInputs[i].nSize); + if (flush_ret != 0) { + std::fprintf(stderr, "AX_SYS_MflushCache failed input=%u ret=0x%x\n", i, + static_cast(flush_ret)); + return flush_ret; + } + } + const AX_S32 ret = AX_ENGINE_RunSync(handle_, &io_); + if (ret != 0) { + std::fprintf(stderr, "AX_ENGINE_RunSync failed ret=0x%x\n", + static_cast(ret)); + return ret; + } + for (AX_U32 i = 0; i < io_.nOutputSize; ++i) { + const AX_S32 invalidate_ret = AX_SYS_MinvalidateCache( + io_.pOutputs[i].phyAddr, io_.pOutputs[i].pVirAddr, + io_.pOutputs[i].nSize); + if (invalidate_ret != 0) { + std::fprintf(stderr, + "AX_SYS_MinvalidateCache failed output=%u ret=0x%x\n", i, + static_cast(invalidate_ret)); + return invalidate_ret; + } + } + return 0; +} + +int EngineWrapper::GetOutputByName(const std::string &name, void *data, + std::size_t size) const { + const int index = OutputIndex(name); + if (!initialized_ || index < 0 || !data) return -1; + const std::size_t expected = io_.pOutputs[index].nSize; + if (size != 0 && size != expected) return -1; + std::memcpy(data, io_.pOutputs[index].pVirAddr, expected); + return 0; +} + +int EngineWrapper::GetInputSizeByName(const std::string &name) const { + const int index = InputIndex(name); + return index < 0 ? -1 : static_cast(io_.pInputs[index].nSize); +} + +int EngineWrapper::GetOutputSizeByName(const std::string &name) const { + const int index = OutputIndex(name); + return index < 0 ? -1 : static_cast(io_.pOutputs[index].nSize); +} + +const std::string &EngineWrapper::InputName(std::size_t index) const { + return input_names_.at(index); +} + +const std::string &EngineWrapper::OutputName(std::size_t index) const { + return output_names_.at(index); +} diff --git a/cpp/src/engine_wrapper.hpp b/cpp/src/engine_wrapper.hpp new file mode 100644 index 0000000000000000000000000000000000000000..0afc550d5a6b027328ea8d7671203c9749cccf00 --- /dev/null +++ b/cpp/src/engine_wrapper.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include + +#include "ax_engine_api.h" + +class EngineWrapper { + public: + EngineWrapper() = default; + ~EngineWrapper(); + + EngineWrapper(const EngineWrapper &) = delete; + EngineWrapper &operator=(const EngineWrapper &) = delete; + + int Init(const std::string &model_path); + void Release(); + int SetInputByName(const std::string &name, const void *data, + std::size_t size = 0); + int ZeroInputByName(const std::string &name); + int RunSync(); + int GetOutputByName(const std::string &name, void *data, + std::size_t size = 0) const; + int GetInputSizeByName(const std::string &name) const; + int GetOutputSizeByName(const std::string &name) const; + const std::string &InputName(std::size_t index) const; + const std::string &OutputName(std::size_t index) const; + + private: + int InputIndex(const std::string &name) const; + int OutputIndex(const std::string &name) const; + + bool initialized_ = false; + AX_ENGINE_HANDLE handle_ = nullptr; + AX_ENGINE_IO_INFO_T *info_ = nullptr; + AX_ENGINE_IO_T io_{}; + std::unordered_map inputs_; + std::unordered_map outputs_; + std::vector input_names_; + std::vector output_names_; +}; diff --git a/cpp/src/wav_reader.hpp b/cpp/src/wav_reader.hpp new file mode 100644 index 0000000000000000000000000000000000000000..ecb607f71990df2827196b98b719ff39f172dfd1 --- /dev/null +++ b/cpp/src/wav_reader.hpp @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +struct PcmWav { + int sample_rate = 0; + std::vector samples; +}; + +inline uint16_t ReadLe16(std::istream &input) { + uint8_t bytes[2]{}; + input.read(reinterpret_cast(bytes), 2); + return static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8); +} + +inline uint32_t ReadLe32(std::istream &input) { + uint8_t bytes[4]{}; + input.read(reinterpret_cast(bytes), 4); + return static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8) | + (static_cast(bytes[2]) << 16) | + (static_cast(bytes[3]) << 24); +} + +inline PcmWav ReadPcmWav(const std::string &path) { + std::ifstream input(path, std::ios::binary); + if (!input) throw std::runtime_error("Cannot open WAV: " + path); + char riff[4]{}; + char wave[4]{}; + input.read(riff, 4); + ReadLe32(input); + input.read(wave, 4); + if (std::memcmp(riff, "RIFF", 4) != 0 || + std::memcmp(wave, "WAVE", 4) != 0) { + throw std::runtime_error("Invalid RIFF/WAVE file: " + path); + } + + uint16_t format = 0; + uint16_t channels = 0; + uint16_t bits = 0; + uint32_t sample_rate = 0; + std::vector data; + while (input && (!format || data.empty())) { + char id[4]{}; + input.read(id, 4); + if (!input) break; + const uint32_t size = ReadLe32(input); + if (std::memcmp(id, "fmt ", 4) == 0) { + if (size < 16) throw std::runtime_error("Invalid WAV fmt chunk"); + format = ReadLe16(input); + channels = ReadLe16(input); + sample_rate = ReadLe32(input); + ReadLe32(input); + ReadLe16(input); + bits = ReadLe16(input); + input.seekg(size - 16, std::ios::cur); + } else if (std::memcmp(id, "data", 4) == 0) { + data.resize(size); + input.read(reinterpret_cast(data.data()), size); + } else { + input.seekg(size, std::ios::cur); + } + if (size & 1U) input.seekg(1, std::ios::cur); + } + if (format != 1 || channels != 1 || bits != 16 || sample_rate == 0 || + data.empty() || data.size() % sizeof(int16_t) != 0) { + throw std::runtime_error("Expected mono 16-bit PCM WAV: " + path); + } + PcmWav result; + result.sample_rate = static_cast(sample_rate); + result.samples.resize(data.size() / sizeof(int16_t)); + std::memcpy(result.samples.data(), data.data(), data.size()); + return result; +} diff --git a/manifest.json b/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3b9d998d2b584d9785de44d7015457c35ccb7259 --- /dev/null +++ b/manifest.json @@ -0,0 +1,217 @@ +{ + "target_hardware": "AX630C/AX650", + "runtime": "axengine", + "axmodel_count": 8, + "files": [ + { + "path": ".gitattributes", + "size_bytes": 179, + "sha256": "7b33e03d32f81ffc4bf98649921ff61fdbdfcff7477345dffc00bd5472351478" + }, + { + "path": ".gitignore", + "size_bytes": 74, + "sha256": "496c11fcbfe22bd803d39235e6539b04e2fc8b0e7d747479302176ae84ae181a" + }, + { + "path": "README.md", + "size_bytes": 3689, + "sha256": "7d609fad6c1e359a68551ce097e64e047056739be92a6311fd701ead738f5cd4" + }, + { + "path": "audio/openwakeword/alexa_test.wav", + "size_bytes": 20044, + "sha256": "915dba88a2b936f3ba3ee91affcdbe9041ffbe532613d3aca58d8a15447fbd81" + }, + { + "path": "audio/openwakeword/hey_jane.wav", + "size_bytes": 75820, + "sha256": "cf1b938bd8ca1a7e9bb042665b9cf92a772a6673c6b98e264b27176ea4d3f443" + }, + { + "path": "audio/openwakeword/hey_mycroft_test.wav", + "size_bytes": 30508, + "sha256": "8d728ce1dbd54a97c6cddf0c80acec3113285407a967368f4abe476119c1a8c6" + }, + { + "path": "config.json", + "size_bytes": 207, + "sha256": "67d68940a1ef8586cad523779ea86876b3bd2362c593c2d4e74ef6dd0557f892" + }, + { + "path": "config/openwakeword_mel_weights.bin", + "size_bytes": 1085608, + "sha256": "f5d3a5bafe0def6c1efb4874d724c272d78edadada6652ef885059db2ba147a7" + }, + { + "path": "config/openwakeword_mel_weights.npz", + "size_bytes": 1086554, + "sha256": "a3de443a7383dc51a4bf8b1d2dc9bbc767f7e33e17af617d993700a5f9947ed6" + }, + { + "path": "cpp/CMakeLists.txt", + "size_bytes": 1948, + "sha256": "89980c1323031f9b1689562f493325b5ad1e1bd7960f85bd1e932ba097d42360" + }, + { + "path": "cpp/README.md", + "size_bytes": 2197, + "sha256": "8b1abab04850f18bbd980b6f4de0f42334bfc1086fac97607606c7f6c914422f" + }, + { + "path": "cpp/bin/openwakeword_ax630c", + "size_bytes": 39688, + "sha256": "309405a685c6928975ec732525ab069316bf43a3495de80c99d6aa566e6c37a3" + }, + { + "path": "cpp/bin/openwakeword_ax650", + "size_bytes": 39688, + "sha256": "cea03dc9b00f40c4508bb82ee8e1dfa2a8775088f4e193fc413ba781e4607984" + }, + { + "path": "cpp/build_ax630c.sh", + "size_bytes": 1299, + "sha256": "4d594ea058d6563fa9177698f0af0e3409959672f399d9729a378de2a47ebc09" + }, + { + "path": "cpp/build_ax650.sh", + "size_bytes": 1267, + "sha256": "fbf962de8ed6433fef6b2aac7332bf84e18720f01b6bf7955e9d32eb409f8e01" + }, + { + "path": "cpp/download_bsp.sh", + "size_bytes": 1290, + "sha256": "0c5afbc290c6d8341df2e8b356626642935a5c9971b0b6b603d30a9b32e69f7a" + }, + { + "path": "cpp/openwakeword_ax.cpp", + "size_bytes": 14122, + "sha256": "6c4ef28769f1321624819a6c915e43d2127989cefc58a39e8600631d57f98d8d" + }, + { + "path": "cpp/run_batch_openwakeword_ax.sh", + "size_bytes": 3938, + "sha256": "5c4a4d3d9e399fb98d0a9b794389fb03ce585cfe9e24d884772052fd1f948c88" + }, + { + "path": "cpp/run_batch_openwakeword_ax630c.sh", + "size_bytes": 168, + "sha256": "f6545c7db37deb52ea727c8f641f000a2c7c1805fc7b40fbc06ec6841b3a259c" + }, + { + "path": "cpp/run_batch_openwakeword_ax650.sh", + "size_bytes": 167, + "sha256": "8711184e6284d9de1ac5b11fdfb23de7c02b9667596d3cb060be702b38b6a858" + }, + { + "path": "cpp/run_openwakeword_ax.sh", + "size_bytes": 1510, + "sha256": "2cf5332a4a6457b1356609fd66e8777d3cc89cc871aa176695cd868d4667f948" + }, + { + "path": "cpp/run_openwakeword_ax630c.sh", + "size_bytes": 162, + "sha256": "f2bd3486e9e8db53835a6d09f9f6a73769342f2a032939842559268342264cc8" + }, + { + "path": "cpp/run_openwakeword_ax650.sh", + "size_bytes": 161, + "sha256": "4ac47b1d3535d6351e595ec8c4594d9a696ae22e4224076f654bb2969f9e7dd8" + }, + { + "path": "cpp/src/engine_wrapper.cpp", + "size_bytes": 6489, + "sha256": "d5ee6d04eaaa6cff492b1b2cd8c60d8519b630b1fc66fd93ee4085cf2121f4ea" + }, + { + "path": "cpp/src/engine_wrapper.hpp", + "size_bytes": 1313, + "sha256": "6a160d748e4fda7c4a69cd2113c36f36c66bf7292e32ffe3df966d24eac51ede" + }, + { + "path": "cpp/src/wav_reader.hpp", + "size_bytes": 2447, + "sha256": "c4890925751e81c78a8429e06fd7fc1e35673d4c9b84989d0f9fd64a364b4d34" + }, + { + "path": "models/630C/.gitkeep", + "size_bytes": 1, + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" + }, + { + "path": "models/650/openwakeword__alexa_v0.1.axmodel", + "size_bytes": 241041, + "sha256": "3595cd242ddf38aca019042d593c45ac1e90107a6d54836ea5593e3dc55f9784" + }, + { + "path": "models/650/openwakeword__embedding_model.axmodel", + "size_bytes": 451068, + "sha256": "a9c579200c4e9c076d86181343cdbe5cf2c786016cd60b333463784b5c2fa9b4" + }, + { + "path": "models/650/openwakeword__hey_jarvis_v0.1.axmodel", + "size_bytes": 365733, + "sha256": "c878a73a9f187505a9496c0b12c479c94de9102d7851bc31a7d75de15a7298ef" + }, + { + "path": "models/650/openwakeword__hey_mycroft_v0.1.axmodel", + "size_bytes": 247553, + "sha256": "1ff391da1aefc04f20df5a24be0d063941459caeef0d5e6d6892c5561049fdf2" + }, + { + "path": "models/650/openwakeword__hey_rhasspy_v0.1.axmodel", + "size_bytes": 64073, + "sha256": "4e13babcf713a3b62ca871bd6d7df50315a09da17ab35c294fa41034baed4a69" + }, + { + "path": "models/650/openwakeword__melspectrogram.axmodel", + "size_bytes": 340028, + "sha256": "9ea2c5293ef3c05c3c5681b947f270d3fe77a0464308d67cf43d9090e9e7fb5f" + }, + { + "path": "models/650/openwakeword__timer_v0.1.axmodel", + "size_bytes": 482100, + "sha256": "4b7c243daadeed2e7ac2b398e5ee13fd7a3c40a5a857ad835809cbfc5d687b5a" + }, + { + "path": "models/650/openwakeword__weather_v0.1.axmodel", + "size_bytes": 320785, + "sha256": "f94fe025de680f18cfb55b7ca8f7cfabb94624f96b9f51424d73e75a9e0b471c" + }, + { + "path": "outputs/.gitkeep", + "size_bytes": 1, + "sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" + }, + { + "path": "reference/local_inference_results.json", + "size_bytes": 15035, + "sha256": "3e4319055ee7b4a7fae93203411acdf71d5596427c63beb2ee07b749ca090a1f" + }, + { + "path": "requirements.txt", + "size_bytes": 12, + "sha256": "a83a2a4ed7ab2e022b9b8a83cfa2b8cf52cc243006c597e8b96f90a8365590ac" + }, + { + "path": "scripts/openwakeword_ax.py", + "size_bytes": 11767, + "sha256": "4ad53f63027b7a650cbd95c6a038cde28d7a047b16cc25b5b54aa4b3b1ec865d" + }, + { + "path": "scripts/openwakeword_ax630c.py", + "size_bytes": 327, + "sha256": "8f00e9bb9edc80dcff413c652eb1878b5174529a6d33a83e1a4f39eab52e6ed2" + }, + { + "path": "scripts/openwakeword_ax650.py", + "size_bytes": 324, + "sha256": "c9b37219849cca038434abeae54d3a5d2f0fdf585126fb8649e7ecffaa643ee4" + }, + { + "path": "scripts/runtime.py", + "size_bytes": 5296, + "sha256": "99b9f762b630a186cf3b010f57b11987564cb76e41ab403e78d3ada57729b0fb" + } + ] +} diff --git a/models/630C/.gitkeep b/models/630C/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/models/630C/.gitkeep @@ -0,0 +1 @@ + diff --git a/models/630C/openwakeword__alexa_v0.1.axmodel b/models/630C/openwakeword__alexa_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..fd85afca862f124796e1a903ed59ee7b33790122 --- /dev/null +++ b/models/630C/openwakeword__alexa_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7115c3e39908999edc3af0dd15baddc6c6e3093f25007a3dcebf18320ef81177 +size 226487 diff --git a/models/630C/openwakeword__embedding_model.axmodel b/models/630C/openwakeword__embedding_model.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..5f9dd95769a94336aadc8040ac0744c3440ed891 --- /dev/null +++ b/models/630C/openwakeword__embedding_model.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dd2250f109865d94aa81c56f0aebdc90ca7dc36c5a80f2c13cce220b9ddd554 +size 376612 diff --git a/models/630C/openwakeword__hey_jarvis_v0.1.axmodel b/models/630C/openwakeword__hey_jarvis_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..5295eb53812a8c0af4d9deca686a7369f4e2e30b --- /dev/null +++ b/models/630C/openwakeword__hey_jarvis_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2d5ab8698daf6ea715b2189a71b0cae3593191780cd29fc703d896aa67b0789 +size 338337 diff --git a/models/630C/openwakeword__hey_mycroft_v0.1.axmodel b/models/630C/openwakeword__hey_mycroft_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..4b875802f7d17bc3bc913794965fc43bcfad36ec --- /dev/null +++ b/models/630C/openwakeword__hey_mycroft_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34627c22ef7aaf915b88516a3a17d18631fe92f8945c734b57ef6b1232307b73 +size 229479 diff --git a/models/630C/openwakeword__hey_rhasspy_v0.1.axmodel b/models/630C/openwakeword__hey_rhasspy_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..f6b8436d97038211a1b68c489373f02400882ea5 --- /dev/null +++ b/models/630C/openwakeword__hey_rhasspy_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:721f58b213a2951e52a80e45e3a1bb96fb761f631157dacb8e5f3aa7591aac26 +size 61607 diff --git a/models/630C/openwakeword__melspectrogram.axmodel b/models/630C/openwakeword__melspectrogram.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..14d744f69cc81bc285787d6ce8fd0ee335e166d8 --- /dev/null +++ b/models/630C/openwakeword__melspectrogram.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3bb40d15f6af7ba3b0aa06702a8bd428ab7a871e28dc6557487fae7cd35e2b7 +size 321576 diff --git a/models/630C/openwakeword__timer_v0.1.axmodel b/models/630C/openwakeword__timer_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..4aeab870d642d7e7ddc8e119e9622c6123fb2e87 --- /dev/null +++ b/models/630C/openwakeword__timer_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:954b46936f4fcee8694379812e0f89d00cea4b8d4f9ec0cb0632ce41a22d3c5b +size 450738 diff --git a/models/630C/openwakeword__weather_v0.1.axmodel b/models/630C/openwakeword__weather_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..b42b8f0ca7f0758a7a1b64266dc8ada647ec9637 --- /dev/null +++ b/models/630C/openwakeword__weather_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e597cc394a4ab34c5b1d9f777d366cb19053b07ec6ff873b163f306fb57a215 +size 300175 diff --git a/models/650/openwakeword__alexa_v0.1.axmodel b/models/650/openwakeword__alexa_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..de1b568de8adf29543e6570c2b58b937275b54bc --- /dev/null +++ b/models/650/openwakeword__alexa_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3595cd242ddf38aca019042d593c45ac1e90107a6d54836ea5593e3dc55f9784 +size 241041 diff --git a/models/650/openwakeword__embedding_model.axmodel b/models/650/openwakeword__embedding_model.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..7e9f784386cc956da6e4a7d2682e244cfb3ed78a --- /dev/null +++ b/models/650/openwakeword__embedding_model.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c579200c4e9c076d86181343cdbe5cf2c786016cd60b333463784b5c2fa9b4 +size 451068 diff --git a/models/650/openwakeword__hey_jarvis_v0.1.axmodel b/models/650/openwakeword__hey_jarvis_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..d5f863b8071776f08e3e1604724358c243bad878 --- /dev/null +++ b/models/650/openwakeword__hey_jarvis_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c878a73a9f187505a9496c0b12c479c94de9102d7851bc31a7d75de15a7298ef +size 365733 diff --git a/models/650/openwakeword__hey_mycroft_v0.1.axmodel b/models/650/openwakeword__hey_mycroft_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..45f2772c958ac9936ea3ad9221e96082f6eee125 --- /dev/null +++ b/models/650/openwakeword__hey_mycroft_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ff391da1aefc04f20df5a24be0d063941459caeef0d5e6d6892c5561049fdf2 +size 247553 diff --git a/models/650/openwakeword__hey_rhasspy_v0.1.axmodel b/models/650/openwakeword__hey_rhasspy_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..dfe0f9453d10b64364d378d128a5fbb1a3a2f348 --- /dev/null +++ b/models/650/openwakeword__hey_rhasspy_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e13babcf713a3b62ca871bd6d7df50315a09da17ab35c294fa41034baed4a69 +size 64073 diff --git a/models/650/openwakeword__melspectrogram.axmodel b/models/650/openwakeword__melspectrogram.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..d081f9be69324591502fbd56a85b3b5645f74ef8 --- /dev/null +++ b/models/650/openwakeword__melspectrogram.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ea2c5293ef3c05c3c5681b947f270d3fe77a0464308d67cf43d9090e9e7fb5f +size 340028 diff --git a/models/650/openwakeword__timer_v0.1.axmodel b/models/650/openwakeword__timer_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..49ab1b34cc4eb43a8187d6bbdae6b1e2c0261269 --- /dev/null +++ b/models/650/openwakeword__timer_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b7c243daadeed2e7ac2b398e5ee13fd7a3c40a5a857ad835809cbfc5d687b5a +size 482100 diff --git a/models/650/openwakeword__weather_v0.1.axmodel b/models/650/openwakeword__weather_v0.1.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..d0d555604768a69299670dacabf4628b22f1230c --- /dev/null +++ b/models/650/openwakeword__weather_v0.1.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f94fe025de680f18cfb55b7ca8f7cfabb94624f96b9f51424d73e75a9e0b471c +size 320785 diff --git a/outputs/.gitkeep b/outputs/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/outputs/.gitkeep @@ -0,0 +1 @@ + diff --git a/outputs/cpp_batch/logs/alexa_test.log b/outputs/cpp_batch/logs/alexa_test.log new file mode 100644 index 0000000000000000000000000000000000000000..c4a0254894315fc00a86a4ab3d5b82b39734e34a --- /dev/null +++ b/outputs/cpp_batch/logs/alexa_test.log @@ -0,0 +1,25 @@ +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__embedding_model.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__alexa_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__hey_jarvis_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__hey_mycroft_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__hey_rhasspy_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__timer_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__weather_v0.1.axmodel (inputs=1 outputs=1) + +openWakeWord C++ inference complete +audio: audio/openwakeword/alexa_test.wav +chunks: 8 +threshold: 0.500 +alexa_v0.1 1.000000 WAKEUP +hey_jarvis_v0.1 0.000005 +hey_mycroft_v0.1 0.000839 +hey_rhasspy_v0.1 0.001408 +timer_v0.1 0.997989 0.011197 0.011197 0.012590 0.011609 0.011197 0.011197 +weather_v0.1 0.010740 +detected: true +audio_seconds: 0.625000 +feature_seconds: 0.038823 +npu_seconds: 0.005308 +model_load_seconds: 0.310009 +inference_seconds: 0.044567 +rtf: 0.071308 diff --git a/outputs/cpp_batch/logs/hey_jane.log b/outputs/cpp_batch/logs/hey_jane.log new file mode 100644 index 0000000000000000000000000000000000000000..a0077b56af28a0c514f4a2a7ec562b07b8c35452 --- /dev/null +++ b/outputs/cpp_batch/logs/hey_jane.log @@ -0,0 +1,25 @@ +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__embedding_model.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__alexa_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__hey_jarvis_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__hey_mycroft_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__hey_rhasspy_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__timer_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__weather_v0.1.axmodel (inputs=1 outputs=1) + +openWakeWord C++ inference complete +audio: audio/openwakeword/hey_jane.wav +chunks: 30 +threshold: 0.500 +alexa_v0.1 0.001312 +hey_jarvis_v0.1 0.017346 +hey_mycroft_v0.1 0.000000 +hey_rhasspy_v0.1 0.004955 +timer_v0.1 1.000000 0.009932 0.008854 0.009936 0.009849 0.008854 0.021674 +weather_v0.1 0.387996 +detected: false +audio_seconds: 2.368000 +feature_seconds: 0.145612 +npu_seconds: 0.022135 +model_load_seconds: 0.653880 +inference_seconds: 0.169402 +rtf: 0.071538 diff --git a/outputs/cpp_batch/logs/hey_mycroft_test.log b/outputs/cpp_batch/logs/hey_mycroft_test.log new file mode 100644 index 0000000000000000000000000000000000000000..3475b10db13e574028bdc71bc9639f42338b8344 --- /dev/null +++ b/outputs/cpp_batch/logs/hey_mycroft_test.log @@ -0,0 +1,25 @@ +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__embedding_model.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__alexa_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__hey_jarvis_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__hey_mycroft_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__hey_rhasspy_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__timer_v0.1.axmodel (inputs=1 outputs=1) +Loaded /mnt/huyuan/workspace/kws_quant/huggingface/openwakeword-axera-inference/models/openwakeword__weather_v0.1.axmodel (inputs=1 outputs=1) + +openWakeWord C++ inference complete +audio: audio/openwakeword/hey_mycroft_test.wav +chunks: 12 +threshold: 0.500 +alexa_v0.1 0.001801 +hey_jarvis_v0.1 0.000025 +hey_mycroft_v0.1 1.000000 WAKEUP +hey_rhasspy_v0.1 0.002780 +timer_v0.1 0.998750 0.010560 0.010560 0.011588 0.011225 0.010560 0.010560 +weather_v0.1 0.279959 +detected: true +audio_seconds: 0.952000 +feature_seconds: 0.058748 +npu_seconds: 0.007945 +model_load_seconds: 0.862838 +inference_seconds: 0.067336 +rtf: 0.070731 diff --git a/outputs/cpp_batch/results.tsv b/outputs/cpp_batch/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..0c431e737d2d8df5c47ef37d0417e54b175d3619 --- /dev/null +++ b/outputs/cpp_batch/results.tsv @@ -0,0 +1,4 @@ +file audio_seconds feature_seconds npu_seconds inference_seconds rtf model_load_seconds detections +alexa_test.wav 0.625000 0.038823 0.005308 0.044567 0.071308 0.310009 alexa_v0.1 +hey_jane.wav 2.368000 0.145612 0.022135 0.169402 0.071538 0.653880 none +hey_mycroft_test.wav 0.952000 0.058748 0.007945 0.067336 0.070731 0.862838 hey_mycroft_v0.1 diff --git a/outputs/cpp_batch/summary.txt b/outputs/cpp_batch/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..0f9d050364f36d7b81d12af3965c5961fdfec77b --- /dev/null +++ b/outputs/cpp_batch/summary.txt @@ -0,0 +1,7 @@ +files: 3 +total_audio_seconds: 3.945000 +total_feature_seconds: 0.243183 +total_npu_seconds: 0.035388 +total_inference_seconds: 0.281305 +total_model_load_seconds: 1.826727 +rtf: 0.071307 diff --git a/reference/local_inference_results.json b/reference/local_inference_results.json new file mode 100644 index 0000000000000000000000000000000000000000..02e984ce5b19d9e90c064e2cebcb0d2bd982caa1 --- /dev/null +++ b/reference/local_inference_results.json @@ -0,0 +1,497 @@ +{ + "environment": { + "python": "3.12.0", + "numpy": "1.26.4", + "onnxruntime": "1.23.1", + "platform": "Linux-5.15.0-94-generic-x86_64-with-glibc2.35" + }, + "openwakeword": { + "pipeline": "fixed 80 ms streaming chunks with 30 ms PCM history", + "clips": [ + { + "audio": "openWakeWord/tests/data/alexa_test.wav", + "sample_rate": 16000, + "sample_count": 10240, + "frame_count": 8, + "initial_context": "480 zero PCM samples for fixed [1,1760] mel input", + "stage_metrics": { + "melspectrogram": { + "shape_match": true, + "comparisons": 8, + "minimum_cosine": 0.9999999999999984, + "maximum_mse": 2.3590018827235326e-11, + "maximum_abs": 1.52587890625e-05 + }, + "embedding": { + "shape_match": true, + "comparisons": 8, + "minimum_cosine": 0.9999999999996391, + "maximum_mse": 2.417571698387633e-10, + "maximum_abs": 3.814697265625e-05 + }, + "classifiers": { + "shape_match": true, + "comparisons": 48, + "minimum_cosine": 0.9999999999999992, + "maximum_mse": 7.993605777301127e-15, + "maximum_abs": 1.1920928955078125e-07 + } + }, + "source_max_scores": { + "alexa_v0.1.onnx": [ + 1.0 + ], + "hey_jarvis_v0.1.onnx": [ + 4.9173831939697266e-06 + ], + "hey_mycroft_v0.1.onnx": [ + 0.0007033348083496094 + ], + "hey_rhasspy_v0.1.onnx": [ + 0.0013944804668426514 + ], + "timer_v0.1.onnx": [ + 0.9979174733161926, + 0.011717854999005795, + 0.011717854999005795, + 0.01298792939633131, + 0.012006871402263641, + 0.011717854999005795, + 0.011717854999005795 + ], + "weather_v0.1.onnx": [ + 0.011630207300186157 + ] + }, + "static_max_scores": { + "alexa_v0.1.onnx": [ + 1.0 + ], + "hey_jarvis_v0.1.onnx": [ + 4.947185516357422e-06 + ], + "hey_mycroft_v0.1.onnx": [ + 0.0007033348083496094 + ], + "hey_rhasspy_v0.1.onnx": [ + 0.0013944804668426514 + ], + "timer_v0.1.onnx": [ + 0.9979174733161926, + 0.011717865243554115, + 0.011717865243554115, + 0.012987940572202206, + 0.012006882578134537, + 0.011717865243554115, + 0.011717865243554115 + ], + "weather_v0.1.onnx": [ + 0.011630237102508545 + ] + }, + "expected_model": "alexa_v0.1.onnx", + "expected_score": 1.0, + "expected_detected": true, + "numerical_match": true, + "passed": true + }, + { + "audio": "openWakeWord/tests/data/hey_mycroft_test.wav", + "sample_rate": 16000, + "sample_count": 15360, + "frame_count": 12, + "initial_context": "480 zero PCM samples for fixed [1,1760] mel input", + "stage_metrics": { + "melspectrogram": { + "shape_match": true, + "comparisons": 12, + "minimum_cosine": 0.9999999999999982, + "maximum_mse": 2.4101609596982598e-11, + "maximum_abs": 1.52587890625e-05 + }, + "embedding": { + "shape_match": true, + "comparisons": 12, + "minimum_cosine": 0.9999999999998549, + "maximum_mse": 1.3607677947409988e-10, + "maximum_abs": 4.1961669921875e-05 + }, + "classifiers": { + "shape_match": true, + "comparisons": 72, + "minimum_cosine": 0.9999999999999998, + "maximum_mse": 1.9454660105111543e-11, + "maximum_abs": 4.410743713378906e-06 + } + }, + "source_max_scores": { + "alexa_v0.1.onnx": [ + 0.0016120076179504395 + ], + "hey_jarvis_v0.1.onnx": [ + 2.256035804748535e-05 + ], + "hey_mycroft_v0.1.onnx": [ + 1.0 + ], + "hey_rhasspy_v0.1.onnx": [ + 0.0026790201663970947 + ], + "timer_v0.1.onnx": [ + 0.9988803267478943, + 0.011138674803078175, + 0.011138674803078175, + 0.012048495002090931, + 0.011720946989953518, + 0.011138674803078175, + 0.011138674803078175 + ], + "weather_v0.1.onnx": [ + 0.18771854043006897 + ] + }, + "static_max_scores": { + "alexa_v0.1.onnx": [ + 0.0016120672225952148 + ], + "hey_jarvis_v0.1.onnx": [ + 2.2590160369873047e-05 + ], + "hey_mycroft_v0.1.onnx": [ + 1.0 + ], + "hey_rhasspy_v0.1.onnx": [ + 0.0026790201663970947 + ], + "timer_v0.1.onnx": [ + 0.9988803267478943, + 0.011138674803078175, + 0.011138674803078175, + 0.012048495002090931, + 0.011720946989953518, + 0.011138674803078175, + 0.011138674803078175 + ], + "weather_v0.1.onnx": [ + 0.18771809339523315 + ] + }, + "expected_model": "hey_mycroft_v0.1.onnx", + "expected_score": 1.0, + "expected_detected": true, + "numerical_match": true, + "passed": true + }, + { + "audio": "openWakeWord/tests/data/hey_jane.wav", + "sample_rate": 16000, + "sample_count": 38400, + "frame_count": 30, + "initial_context": "480 zero PCM samples for fixed [1,1760] mel input", + "stage_metrics": { + "melspectrogram": { + "shape_match": true, + "comparisons": 30, + "minimum_cosine": 0.9999999999999951, + "maximum_mse": 1.836042429204099e-11, + "maximum_abs": 1.52587890625e-05 + }, + "embedding": { + "shape_match": true, + "comparisons": 30, + "minimum_cosine": 0.9999999999996753, + "maximum_mse": 1.5618928372873597e-10, + "maximum_abs": 3.910064697265625e-05 + }, + "classifiers": { + "shape_match": true, + "comparisons": 180, + "minimum_cosine": 0.0, + "maximum_mse": 4.3520742565306136e-14, + "maximum_abs": 2.086162567138672e-07 + } + }, + "source_max_scores": { + "alexa_v0.1.onnx": [ + 0.001211315393447876 + ], + "hey_jarvis_v0.1.onnx": [ + 0.0183732807636261 + ], + "hey_mycroft_v0.1.onnx": [ + 3.1888484954833984e-06 + ], + "hey_rhasspy_v0.1.onnx": [ + 0.0046852827072143555 + ], + "timer_v0.1.onnx": [ + 0.9999986886978149, + 0.01402811985462904, + 0.009387511759996414, + 0.010385164059698582, + 0.012926612980663776, + 0.009387511759996414, + 0.03614084795117378 + ], + "weather_v0.1.onnx": [ + 0.31700584292411804 + ] + }, + "static_max_scores": { + "alexa_v0.1.onnx": [ + 0.001211315393447876 + ], + "hey_jarvis_v0.1.onnx": [ + 0.018373221158981323 + ], + "hey_mycroft_v0.1.onnx": [ + 3.159046173095703e-06 + ], + "hey_rhasspy_v0.1.onnx": [ + 0.0046852827072143555 + ], + "timer_v0.1.onnx": [ + 0.9999986886978149, + 0.01402811985462904, + 0.009387511759996414, + 0.010385164059698582, + 0.012926612980663776, + 0.009387511759996414, + 0.036140866577625275 + ], + "weather_v0.1.onnx": [ + 0.31700563430786133 + ] + }, + "expected_model": null, + "expected_score": null, + "expected_detected": null, + "numerical_match": true, + "passed": true + } + ], + "passed": true + }, + "sherpa": { + "sherpa_onnx_version": "1.13.4", + "variants": [ + { + "chunk_size": 8, + "max_active_paths": 1, + "audio_count": 9, + "comparisons": [ + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/en_0.wav", + "source_detections": [ + "LIGHT_UP" + ], + "static_detections": [ + "LIGHT_UP" + ], + "source_decode_calls": 45, + "static_decode_calls": 45, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/en_1.wav", + "source_detections": [ + "LOVELY_CHILD" + ], + "static_detections": [ + "LOVELY_CHILD" + ], + "source_decode_calls": 108, + "static_decode_calls": 108, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_0.wav", + "source_detections": [], + "static_detections": [], + "source_decode_calls": 39, + "static_decode_calls": 39, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_1.wav", + "source_detections": [], + "static_detections": [], + "source_decode_calls": 36, + "static_decode_calls": 36, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_2.wav", + "source_detections": [], + "static_detections": [], + "source_decode_calls": 32, + "static_decode_calls": 32, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_3.wav", + "source_detections": [ + "法国" + ], + "static_detections": [ + "法国" + ], + "source_decode_calls": 54, + "static_decode_calls": 54, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_4.wav", + "source_detections": [ + "蒋友伯", + "女儿" + ], + "static_detections": [ + "蒋友伯", + "女儿" + ], + "source_decode_calls": 32, + "static_decode_calls": 32, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_5.wav", + "source_detections": [ + "周望军", + "落实" + ], + "static_detections": [ + "周望军", + "落实" + ], + "source_decode_calls": 30, + "static_decode_calls": 30, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_6.wav", + "source_detections": [ + "朱丽楠" + ], + "static_detections": [ + "朱丽楠" + ], + "source_decode_calls": 26, + "static_decode_calls": 26, + "match": true + } + ], + "detected_audio_count": 6, + "passed": true + }, + { + "chunk_size": 16, + "max_active_paths": 1, + "audio_count": 9, + "comparisons": [ + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/en_0.wav", + "source_detections": [], + "static_detections": [], + "source_decode_calls": 22, + "static_decode_calls": 22, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/en_1.wav", + "source_detections": [ + "LOVELY_CHILD" + ], + "static_detections": [ + "LOVELY_CHILD" + ], + "source_decode_calls": 54, + "static_decode_calls": 54, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_0.wav", + "source_detections": [], + "static_detections": [], + "source_decode_calls": 19, + "static_decode_calls": 19, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_1.wav", + "source_detections": [], + "static_detections": [], + "source_decode_calls": 18, + "static_decode_calls": 18, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_2.wav", + "source_detections": [], + "static_detections": [], + "source_decode_calls": 16, + "static_decode_calls": 16, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_3.wav", + "source_detections": [ + "法国" + ], + "static_detections": [ + "法国" + ], + "source_decode_calls": 27, + "static_decode_calls": 27, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_4.wav", + "source_detections": [ + "蒋友伯", + "女儿" + ], + "static_detections": [ + "蒋友伯", + "女儿" + ], + "source_decode_calls": 16, + "static_decode_calls": 16, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_5.wav", + "source_detections": [ + "周望军", + "落实" + ], + "static_detections": [ + "周望军", + "落实" + ], + "source_decode_calls": 15, + "static_decode_calls": 15, + "match": true + }, + { + "audio": "sherpa-onnx-kws-zipformer-zh-en-3M-2025-12-20/test_wavs/zh_6.wav", + "source_detections": [ + "朱丽楠" + ], + "static_detections": [ + "朱丽楠" + ], + "source_decode_calls": 13, + "static_decode_calls": 13, + "match": true + } + ], + "detected_audio_count": 5, + "passed": true + } + ], + "passed": true + }, + "elapsed_seconds": 9.052458902820945, + "all_passed": true +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9f161acad9345ad485c69c3900c8cbe263a3a9bd --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +numpy>=1.24 diff --git a/scripts/__pycache__/openwakeword_ax.cpython-312.pyc b/scripts/__pycache__/openwakeword_ax.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4658ab821d708e3928b58a520314fa2eda982c0e Binary files /dev/null and b/scripts/__pycache__/openwakeword_ax.cpython-312.pyc differ diff --git a/scripts/__pycache__/openwakeword_ax650.cpython-312.pyc b/scripts/__pycache__/openwakeword_ax650.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..193e5ad74962edb83d3e5b29744be0ee7853685a Binary files /dev/null and b/scripts/__pycache__/openwakeword_ax650.cpython-312.pyc differ diff --git a/scripts/__pycache__/runtime.cpython-312.pyc b/scripts/__pycache__/runtime.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cdbd0c7052ec4c8ad66c163e7a3dbe9f9bc2bb63 Binary files /dev/null and b/scripts/__pycache__/runtime.cpython-312.pyc differ diff --git a/scripts/openwakeword_ax.py b/scripts/openwakeword_ax.py new file mode 100644 index 0000000000000000000000000000000000000000..63859d6e1325bc601237a96d9f4b84e8ba19f306 --- /dev/null +++ b/scripts/openwakeword_ax.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import platform +import time +from pathlib import Path +from typing import Any + +import numpy as np + +from runtime import InferenceSession, first_output, load_json, max_abs_delta, read_wav, write_json + + +SCRIPT_ROOT = Path(__file__).resolve().parents[1] +REPO_LAYOUT = (SCRIPT_ROOT / "model_convert/pulsar2/check0/axmodels").is_dir() +DEFAULT_MODELS_DIR = ( + SCRIPT_ROOT / "model_convert/pulsar2/check0/axmodels" + if REPO_LAYOUT + else SCRIPT_ROOT / "models/650" +) +DEFAULT_AUDIO_DIR = ( + SCRIPT_ROOT / "openWakeWord/tests/data" + if REPO_LAYOUT + else SCRIPT_ROOT / "audio/openwakeword" +) +DEFAULT_REFERENCE = ( + SCRIPT_ROOT / "model_convert/local_inference/results.json" + if REPO_LAYOUT + else SCRIPT_ROOT / "reference/local_inference_results.json" +) +DEFAULT_MEL_WEIGHTS = ( + SCRIPT_ROOT / "model_convert/board_assets/openwakeword_mel_weights.npz" + if REPO_LAYOUT + else SCRIPT_ROOT / "config/openwakeword_mel_weights.npz" +) +DEFAULT_OUTPUT = ( + SCRIPT_ROOT / "model_convert/board_inference/openwakeword_board.json" + if REPO_LAYOUT + else SCRIPT_ROOT / "outputs/openwakeword_ax650_board.json" +) +CLASSIFIERS = ( + "alexa_v0.1", + "hey_jarvis_v0.1", + "hey_mycroft_v0.1", + "hey_rhasspy_v0.1", + "timer_v0.1", + "weather_v0.1", +) +EXPECTED = { + "alexa_test.wav": "alexa_v0.1", + "hey_mycroft_test.wav": "hey_mycroft_v0.1", +} + + +def mel_host_postprocess(value: np.ndarray) -> np.ndarray: + clipped = np.asarray(value, dtype=np.float32) + decibels = np.log(clipped).astype(np.float32) + decibels = decibels * np.float32(10.0) + decibels = decibels / np.float32(2.3025851249694824) + minimum = np.max(decibels).astype(np.float32) - np.float32(80.0) + return np.maximum(decibels, minimum).astype(np.float32) + + +def load_mel_weights(path: Path) -> dict[str, np.ndarray]: + if not path.is_file(): + raise FileNotFoundError(path) + with np.load(path) as values: + result = { + "real": np.asarray(values["real"], dtype=np.float32), + "imag": np.asarray(values["imag"], dtype=np.float32), + "mel": np.asarray(values["mel"], dtype=np.float32), + "floor": np.asarray(values["floor"], dtype=np.float32).reshape(()), + } + expected_shapes = { + "real": (257, 512), + "imag": (257, 512), + "mel": (257, 32), + } + for name, shape in expected_shapes.items(): + if result[name].shape != shape: + raise ValueError(f"unexpected mel weight shape for {name}: {result[name].shape}") + return result + + +def numpy_melspectrogram( + value: np.ndarray, weights: dict[str, np.ndarray] +) -> np.ndarray: + samples = np.asarray(value, dtype=np.float32).reshape(-1) + if samples.shape != (1760,): + raise ValueError(f"expected 1760 mel input samples, got {samples.shape}") + frames = np.stack( + [samples[start : start + 512] for start in range(0, 1280, 160)], + axis=0, + ) + real = frames @ weights["real"].T + imag = frames @ weights["imag"].T + power = np.square(real) + np.square(imag) + mel = power @ weights["mel"] + mel = np.maximum(mel, weights["floor"]).astype(np.float32) + return mel[None, None, :, :] + + +def model_path(models_dir: Path, name: str, backend: str) -> Path: + if backend == "axengine": + return models_dir / f"openwakeword__{name}.axmodel" + return models_dir / f"{name}.onnx" + + +def pad_chunks(samples: np.ndarray, chunk_size: int = 1280) -> np.ndarray: + remainder = samples.size % chunk_size + if remainder == 0: + return samples + return np.pad(samples, (0, chunk_size - remainder)) + + +def load_sessions( + models_dir: Path, backend: str, mel_backend: str +) -> dict[str, InferenceSession]: + names = ("embedding_model",) + CLASSIFIERS + if mel_backend == "model": + names = ("melspectrogram",) + names + sessions = { + name: InferenceSession(model_path(models_dir, name, backend), backend) + for name in names + } + for name, session in sessions.items(): + print( + f"loaded {name}: " + f"inputs={[(value.name, value.shape, str(value.dtype)) for value in session.inputs]} " + f"outputs={[(value.name, value.shape, str(value.dtype)) for value in session.outputs]}", + flush=True, + ) + return sessions + + +def reference_by_audio(path: Path | None) -> dict[str, dict[str, Any]]: + if path is None or not path.is_file(): + return {} + result = load_json(path) + return { + Path(row["audio"]).name: row + for row in result["openwakeword"]["clips"] + } + + +def infer_clip( + audio_path: Path, + sessions: dict[str, InferenceSession], + reference: dict[str, Any] | None, + threshold: float, + score_tolerance: float, + mel_backend: str, + mel_weights: dict[str, np.ndarray] | None, +) -> dict[str, Any]: + samples, sample_rate = read_wav(audio_path) + if sample_rate != 16000: + raise ValueError(f"expected 16 kHz audio: {audio_path} has {sample_rate}") + samples = pad_chunks(samples) + + history = np.zeros(480, dtype=np.int16) + mel_buffer = np.ones((76, 32), dtype=np.float32) + feature_buffer = np.zeros((34, 96), dtype=np.float32) + scores: dict[str, list[list[float]]] = {name: [] for name in CLASSIFIERS} + + for start in range(0, samples.size, 1280): + chunk = samples[start : start + 1280] + mel_input = np.concatenate((history, chunk)).astype(np.float32)[None, :] + history = np.concatenate((history, chunk))[-480:].astype(np.int16, copy=False) + + if mel_backend == "numpy": + if mel_weights is None: + raise RuntimeError("NumPy mel backend requires mel weights") + mel_output = numpy_melspectrogram(mel_input, mel_weights) + else: + mel_session = sessions["melspectrogram"] + mel_output = first_output( + mel_session, + mel_session.run({mel_session.inputs[0].name: mel_input}), + ) + mel_output = mel_host_postprocess(mel_output) + spec = np.squeeze(mel_output).astype(np.float32) / 10.0 + 2.0 + if spec.ndim != 2 or spec.shape[1] != 32: + raise RuntimeError(f"unexpected mel output shape: {spec.shape}") + mel_buffer = np.vstack((mel_buffer, spec))[-970:] + + embedding_input = mel_buffer[-76:][None, :, :, None].astype(np.float32) + embedding_session = sessions["embedding_model"] + embedding_output = first_output( + embedding_session, + embedding_session.run( + {embedding_session.inputs[0].name: embedding_input} + ), + ) + feature = np.asarray(embedding_output).reshape(-1, 96)[-1] + feature_buffer = np.vstack((feature_buffer, feature))[-120:] + + for name in CLASSIFIERS: + classifier = sessions[name] + frame_count = classifier.inputs[0].shape[1] + classifier_input = feature_buffer[-frame_count:][None, :, :].astype( + np.float32 + ) + output = first_output( + classifier, + classifier.run({classifier.inputs[0].name: classifier_input}), + ) + scores[name].append(np.asarray(output).reshape(-1).tolist()) + + max_scores = { + name: np.max(np.asarray(values, dtype=np.float64), axis=0).tolist() + for name, values in scores.items() + } + expected_model = EXPECTED.get(audio_path.name) + expected_score = ( + float(np.max(max_scores[expected_model])) if expected_model is not None else None + ) + expected_detected = ( + expected_score >= threshold if expected_score is not None else None + ) + + reference_deltas = None + maximum_reference_delta = None + reference_match = None + if reference is not None: + reference_scores = { + name.removesuffix(".onnx"): value + for name, value in reference["static_max_scores"].items() + } + reference_deltas = { + name: max_abs_delta(reference_scores[name], max_scores[name]) + for name in CLASSIFIERS + } + maximum_reference_delta = max(reference_deltas.values()) + reference_match = maximum_reference_delta <= score_tolerance + + functional_pass = expected_detected is not False + passed = functional_pass and reference_match is not False + return { + "audio": str(audio_path), + "sample_rate": sample_rate, + "sample_count": int(samples.size), + "frame_count": int(samples.size // 1280), + "max_scores": max_scores, + "expected_model": expected_model, + "threshold": threshold, + "expected_score": expected_score, + "expected_detected": expected_detected, + "reference_deltas": reference_deltas, + "maximum_reference_delta": maximum_reference_delta, + "score_tolerance": score_tolerance, + "reference_match": reference_match, + "passed": passed, + } + + +def main( + default_models_dir: Path = DEFAULT_MODELS_DIR, + default_output: Path = DEFAULT_OUTPUT, + target_hardware: str = "AX650", +) -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--backend", choices=("axengine", "onnx"), default="axengine") + parser.add_argument("--mel-backend", choices=("numpy", "model"), default="numpy") + parser.add_argument("--mel-weights", type=Path, default=DEFAULT_MEL_WEIGHTS) + parser.add_argument("--models-dir", type=Path, default=default_models_dir) + parser.add_argument("--audio-dir", type=Path, default=DEFAULT_AUDIO_DIR) + parser.add_argument( + "--reference", + type=Path, + default=DEFAULT_REFERENCE, + ) + parser.add_argument( + "--output", type=Path, default=default_output + ) + parser.add_argument("--threshold", type=float, default=0.5) + parser.add_argument("--score-tolerance", type=float, default=0.15) + args = parser.parse_args() + + audio_files = sorted(args.audio_dir.glob("*.wav")) + if not audio_files: + raise FileNotFoundError(f"no WAV files found under {args.audio_dir}") + + references = reference_by_audio(args.reference) + mel_weights = load_mel_weights(args.mel_weights) if args.mel_backend == "numpy" else None + sessions = load_sessions(args.models_dir, args.backend, args.mel_backend) + started = time.perf_counter() + clips = [] + for path in audio_files: + clip = infer_clip( + path, + sessions, + references.get(path.name), + args.threshold, + args.score_tolerance, + args.mel_backend, + mel_weights, + ) + clips.append(clip) + print( + f"{path.name}: expected={clip['expected_model']} " + f"score={clip['expected_score']} detected={clip['expected_detected']} " + f"reference_delta={clip['maximum_reference_delta']} " + f"passed={clip['passed']}", + flush=True, + ) + + result = { + "backend": args.backend, + "target_hardware": target_hardware, + "platform": platform.platform(), + "models_dir": str(args.models_dir), + "mel_backend": args.mel_backend, + "mel_weights": str(args.mel_weights) if mel_weights is not None else None, + "audio_dir": str(args.audio_dir), + "reference": str(args.reference) if args.reference else None, + "elapsed_seconds": time.perf_counter() - started, + "clips": clips, + "all_passed": all(clip["passed"] for clip in clips), + } + write_json(args.output, result) + print(f"wrote {args.output}") + print(f"all_passed={result['all_passed']}") + if not result["all_passed"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/openwakeword_ax630c.py b/scripts/openwakeword_ax630c.py new file mode 100644 index 0000000000000000000000000000000000000000..26bd41a9b90e62ca2851789df8257ee1ddb9d8ac --- /dev/null +++ b/scripts/openwakeword_ax630c.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +from pathlib import Path + +from openwakeword_ax import main + + +ROOT = Path(__file__).resolve().parents[1] + + +if __name__ == "__main__": + main( + default_models_dir=ROOT / "models/630C", + default_output=ROOT / "outputs/openwakeword_ax630c_board.json", + target_hardware="AX630C", + ) diff --git a/scripts/openwakeword_ax650.py b/scripts/openwakeword_ax650.py new file mode 100644 index 0000000000000000000000000000000000000000..afdab7a2159b5f6c8cf112065a37060781d2e7aa --- /dev/null +++ b/scripts/openwakeword_ax650.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +from pathlib import Path + +from openwakeword_ax import main + + +ROOT = Path(__file__).resolve().parents[1] + + +if __name__ == "__main__": + main( + default_models_dir=ROOT / "models/650", + default_output=ROOT / "outputs/openwakeword_ax650_board.json", + target_hardware="AX650", + ) diff --git a/scripts/runtime.py b/scripts/runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..9702fc45bbbbbfa6d748f90932e3b77604a10b94 --- /dev/null +++ b/scripts/runtime.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import wave +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + + +@dataclass(frozen=True) +class TensorInfo: + name: str + shape: tuple[int, ...] + dtype: np.dtype + + +def _numpy_dtype(value: Any) -> np.dtype: + text = str(value).lower() + mapping = ( + (("tensor(float)", "float32", "fp32", "f32"), np.float32), + (("tensor(float16)", "float16", "fp16", "f16"), np.float16), + (("tensor(int64)", "int64", "s64"), np.int64), + (("tensor(int32)", "int32", "s32"), np.int32), + (("tensor(int16)", "int16", "s16"), np.int16), + (("tensor(int8)", "int8", "s8"), np.int8), + (("tensor(uint16)", "uint16", "u16"), np.uint16), + (("tensor(uint8)", "uint8", "u8"), np.uint8), + (("tensor(bool)", "bool"), np.bool_), + ) + for aliases, dtype in mapping: + if any(alias in text for alias in aliases): + return np.dtype(dtype) + raise ValueError(f"unsupported runtime tensor dtype: {value}") + + +def _tensor_info(value: Any) -> TensorInfo: + shape = getattr(value, "shape", None) + if shape is None: + shape = getattr(value, "dims", None) + dtype = getattr(value, "dtype", None) + if dtype is None: + dtype = getattr(value, "type", None) + return TensorInfo( + name=value.name, + shape=tuple(int(dim) for dim in shape), + dtype=_numpy_dtype(dtype), + ) + + +class InferenceSession: + """Small common wrapper for ONNX Runtime and AXEngine.""" + + def __init__(self, model_path: str | Path, backend: str): + self.path = Path(model_path) + if not self.path.is_file(): + raise FileNotFoundError(self.path) + self.backend = backend + if backend == "axengine": + try: + import axengine + except ImportError as error: + raise RuntimeError( + "axengine is unavailable; run this backend on an AXERA board" + ) from error + self._session = axengine.InferenceSession(str(self.path)) + elif backend == "onnx": + try: + import onnxruntime as ort + except ImportError as error: + raise RuntimeError("onnxruntime is required for --backend onnx") from error + options = ort.SessionOptions() + options.inter_op_num_threads = 1 + options.intra_op_num_threads = 1 + self._session = ort.InferenceSession( + str(self.path), + sess_options=options, + providers=["CPUExecutionProvider"], + ) + else: + raise ValueError(f"unsupported backend: {backend}") + + self.inputs = [_tensor_info(value) for value in self._session.get_inputs()] + self.outputs = [_tensor_info(value) for value in self._session.get_outputs()] + self.input_by_name = {value.name: value for value in self.inputs} + + def run(self, feed: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + missing = [value.name for value in self.inputs if value.name not in feed] + if missing: + raise KeyError(f"missing inputs for {self.path.name}: {missing}") + prepared = { + name: np.ascontiguousarray(np.asarray(feed[name], dtype=meta.dtype)) + for name, meta in self.input_by_name.items() + } + values = self._session.run(None, prepared) + if isinstance(values, dict): + return {name: np.asarray(value) for name, value in values.items()} + if not isinstance(values, (list, tuple)): + values = [values] + if len(values) != len(self.outputs): + raise RuntimeError( + f"unexpected output count from {self.path.name}: " + f"{len(values)} != {len(self.outputs)}" + ) + return { + meta.name: np.asarray(value) + for meta, value in zip(self.outputs, values) + } + + +def first_output(session: InferenceSession, outputs: dict[str, np.ndarray]) -> np.ndarray: + return outputs[session.outputs[0].name] + + +def read_wav(path: str | Path) -> tuple[np.ndarray, int]: + path = Path(path) + with wave.open(str(path), "rb") as source: + if source.getnchannels() != 1 or source.getsampwidth() != 2: + raise ValueError(f"expected mono 16-bit PCM WAV: {path}") + sample_rate = source.getframerate() + samples = np.frombuffer( + source.readframes(source.getnframes()), dtype=np.int16 + ).copy() + return samples, sample_rate + + +def load_json(path: str | Path) -> dict[str, Any]: + return json.loads(Path(path).read_text()) + + +def write_json(path: str | Path, value: Any) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") + + +def max_abs_delta(reference: Any, candidate: Any) -> float: + reference_array = np.asarray(reference, dtype=np.float64) + candidate_array = np.asarray(candidate, dtype=np.float64) + if reference_array.shape != candidate_array.shape: + return float("inf") + if reference_array.size == 0: + return 0.0 + return float(np.max(np.abs(reference_array - candidate_array)))