diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..9b7e40e21f75ac65b3c7896a4749396159cc07a1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,31 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_LEFT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_RIGHT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_LEFT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_RIGHT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_LEFT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_RIGHT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_LEFT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_RIGHT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_LEFT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_RIGHT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_LEFT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_RIGHT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_LEFT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_RIGHT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_LEFT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_RIGHT.jpg filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/models/QCS8550/FP16/backbone_context.bin.aidem filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/models/QCS8550/FP16/decoder_context.bin.aidem filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/models/QCS8550/FP16/scene_start_encoder_context.bin.aidem filter=lfs diff=lfs merge=lfs -text +targets/QCS8550/FP16/models/QCS8550/FP16/temporal_encoder_context.bin.aidem filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE_MODEL_FARM.pdf b/LICENSE_MODEL_FARM.pdf new file mode 100644 index 0000000000000000000000000000000000000000..93e40dd32640b14c6bbe76ea3daacacbef8ca730 Binary files /dev/null and b/LICENSE_MODEL_FARM.pdf differ diff --git a/NOTICE.md b/NOTICE.md new file mode 100644 index 0000000000000000000000000000000000000000..1db63c05b87840dd6a6c980becb1a45c188f9c9b --- /dev/null +++ b/NOTICE.md @@ -0,0 +1,5 @@ +# Provenance + +Resources are copied without binary changes from the APLUX Model Farm build `registry/BEVFormer-tiny (R50)` that Agora verified on real boards (decoded sample check, then measured), pinned by sha256 in SHA256SUMS. + +Agora arranges files under targets/// and adds model.json. AidLite, AidGen, AidVoice and the Qualcomm QNN runtime are separate dependencies with their own terms. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dbb77be3e816ebf666f6bf230e1cff00eea6a033 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +--- +license: other +license_name: aplux-model-farm +license_link: https://huggingface.co/APLUX-Agora/BEVFormer-tiny-R50/blob/main/LICENSE_MODEL_FARM.pdf +tags: [Agora, QNN, AIoT, Qualcomm] +--- + +# BEVFormer-tiny (R50) model resources + +输入六路相机与车辆标定，使用 NPU 进行鸟瞰空间编码和三维目标检测；支持显式上一帧状态。 + +Board builds that [Agora](https://mega.aidlux.com) runs and measures on real Qualcomm boards. Try it online, without a board: https://mega.aidlux.com/models/bevformer-tiny-r50 + +| SoC · board | Precisions | Measured on Agora (default precision) | +| --- | --- | --- | +| QCS8550 · 犀牛派 X1 (Rhino Pi X1) | FP16 | 402.1 ms | + +Task: 其他. Inputs: front (image), front_right (image), front_left (image), back (image), back_left (image), back_right (image), calibration (text), history (json). Outputs: image (image), out (json), history (json). + +## Download + +```bash +export AGORA_WORKSPACE="$HOME/my-bevformer-tiny-r50" +export AGORA_SOC='QCS8550' +hf download APLUX-Agora/BEVFormer-tiny-R50 \ + --include "targets/$AGORA_SOC/*" --local-dir "$AGORA_WORKSPACE/resources" +``` + +Each `targets//` holds `model.json` (default precision and variant paths) and one directory per precision. Context binaries are named `model.bin`; multi-file packages are kept as published by Model Farm. + +Pre- and post-processing, the runner and an exported project: open the model on Agora and choose “部署指南”, or export any workflow that uses it. + +## Licence and provenance + +Source model licence: see the original model's terms. +Converted, deployable resources: APLUX Model Farm License (LICENSE_MODEL_FARM.pdf). See NOTICE.md and SHA256SUMS. diff --git a/SHA256SUMS b/SHA256SUMS new file mode 100644 index 0000000000000000000000000000000000000000..0ad4bbd927d06f052148db0f17caa2b40fbc2369 --- /dev/null +++ b/SHA256SUMS @@ -0,0 +1,60 @@ +35def1b2da02f8d1b3f5b5dcbb40fb74ddd5074f6dc16f9dd2398509d0d0e6d0 targets/QCS8550/FP16/calibration/sample_000.json +d37b43abb24770364bc0f344d1208e5256de9a3f85c5f9b6e3f6cf18db1d094b targets/QCS8550/FP16/calibration/sample_001.json +89c891d43cc43e3665e32fec157b861d12d8fd38324a4ae7a5a8350290ef7c83 targets/QCS8550/FP16/calibration/sample_002.json +80052246ff0d259bbcb0b719a0a0c86909808a6f993b94999deef26085ef0920 targets/QCS8550/FP16/calibration/sample_003.json +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 targets/QCS8550/FP16/code/.gitkeep +1555a8c1d29dfde3c920830faca6eda8cbc6ba36eaf0888379de6a845336380c targets/QCS8550/FP16/code/README.md +3b3f71b3b61a91a85b63a6b8c7257437943fcd97fef6b2b453ffcb7a50d0a0ae targets/QCS8550/FP16/code/python/bevformer.py +bfe8933e69529b417012db2bc5c01a8daad3b86240a269f08eb8fc27f0bfc689 targets/QCS8550/FP16/code/python/camera_grid_visualization.py +2be061c394e1bf4054d6eed07d166f1d5f97502c91b5c679d47943b378a26494 targets/QCS8550/FP16/code/python/configs/demo_config.json +492eb30a0051e11b5bbbfd454c0c4eb9fb27e50259a9651866341e90f2b9ffd6 targets/QCS8550/FP16/code/python/configs/nms_runtime_contract.json +e50d0fc8c46c0ba42578a508d5c9acba16345dc77d172567cd8c0400199c7cab targets/QCS8550/FP16/code/python/datasets/sample4/README.md +14a9f01da4dc57ac2f57e43e9a68a9ee10c5c38352e721bf2f0665b103f5c9aa targets/QCS8550/FP16/code/python/datasets/sample4/asset_manifest.json +047dd43f3f9fb7cc426b1d4ae369181242c5555b84baff5d7e1cbd0dc8b07259 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK.jpg +27b2826991733662e4f17c509ad5a66e85e0ca66a6413063d8db9645fa608c11 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_LEFT.jpg +a998caf0e327816b82d2a142d2581a8c1f9c4e67d50b4af3718a381337b7e415 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_RIGHT.jpg +0e2067a1618c4424050c45c393ada8c9dbf130fff8d6015004fce901dc07984d targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT.jpg +8c52f73a3ef06a129c27ef51ac65704428caebfaa60c52f14a5953bbb0124a18 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_LEFT.jpg +15cf59c79b086d14da6fae5a7ee5b0f4d0fac0c0a22c0a85faaa5fb215521191 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_RIGHT.jpg +974730a9cf60a4a0cae487ed4b296faf6621abec0bcd7276020ade042fd8dc3b targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/can_bus.raw +f1df04f672d707f2fbc8ea9799ee84d8795a1fe22f17ed89c08327e63f663501 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/lidar2img.raw +fdbdaa77e60ed5a6a762a4ea9a8eb1cb2ca9a3d94c610a55a2399a05c04b6ba6 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK.jpg +000f7943769e91ffa842a7dfd1bfc31e0c3f4b37325a98846e8261304028f2ad targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_LEFT.jpg +39d28a8a5a6f61370a7e7e1e965a6d8610b571195c50fd29da960e1fb2fbee80 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_RIGHT.jpg +d5624fcecf12310c3d0760362e097c38cdcdce987e032aaf2883071cd14e2732 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT.jpg +0fe15c0581db41e084df27acb2183b0ab9b0fd36da136ef1b4bcbda6f1a84c58 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_LEFT.jpg +dd64d7463bc06bfd5d5378bdd494b40d68d0894307f3ed971eaaaf131022cf68 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_RIGHT.jpg +8ece2acd67155365b9394e1569b8242d69f3d9805cea19a5153cb51e79e55152 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/can_bus.raw +63d7bcbf3a9c63e1184947d9e5112fe30bf4e8533ea29df44daeef3cb610a2ed targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/lidar2img.raw +741d75aad896829e7246d446635bc21331242c7e28ef869f986d70e048d40962 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/rotation_can_bus.raw +530492c17f13ce48ec402622565d7a4ba0987a57b988a3c4d824cc4987dfd5fc targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/shift.raw +a9a05b52aea16ccf730447170eb836697c26928a112a74249caf423d91d28306 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK.jpg +a28438e9ff68ccab2ce3a8da4151591a2b15780aecdc21185a911318ed923f72 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_LEFT.jpg +1578ba72afd2cc91abb1295c3404a432e41ffc7385b4031a43b586acfb693872 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_RIGHT.jpg +0bb82b89f9e6ce47d9e1bcd747b01fe3fd342ccc67eb7a9c693a3fc3fc73a6bc targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT.jpg +4782aaa9180ef1fe153e702031a3249ada09bdb056ad13ff69abaa2f9e59e35a targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_LEFT.jpg +70856b9d713c279b4b3d5911be30944f339760cc81da5a8f2c4d90a20276c1d3 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_RIGHT.jpg +e8badd418a0eb1905986c2a2a5e87fa6e16532dc4899cfff93f47dbe51bed9be targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/can_bus.raw +7d524fefbeec4c1014ab2773b722aaa13774cb4787fb950e27b1b4de7cc3d403 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/lidar2img.raw +a08e149204cd22c460e8b5cc62726de1b29d22fb05b6c33b450e1f94a162d3c1 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/rotation_can_bus.raw +9cd32a1a7fbb67a9841ebee6acf497db1fa6142ec9e3ec5c9bb1596dd8be81e8 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/shift.raw +34e84e10e04ab229864bd0ddd3067442c0b58e249eeb48088ab417cb259b4985 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK.jpg +cdf1f064c61add58e17d7011713aebbe40be9d7e8290c8816f585fca3841e033 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_LEFT.jpg +eae354ba8407662caa6c50934b6ee306a4e5aaaeec0b0f2e40c2f6b481485d3f targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_RIGHT.jpg +54384bc8c4896f349bdc643b573f398fbb03ee842ac2af2af2c602a62f48f79d targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT.jpg +1f20956c425bdc84da1bf2610b7214d48930652782394aa0f1f3bef94ce0be35 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_LEFT.jpg +4c5209d28a7e3a5f29bd7c02c29bd0f53c669afc43b17af2db098371a9711dc1 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_RIGHT.jpg +dbc4945c65d5804131f75f04578e3fbc288330ff5fa977f224eda07eb96e463a targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/can_bus.raw +64cd7874438918c64f54414d1b873b0b0fcf05b2b601f2014c4fe5a8316e7a2c targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/lidar2img.raw +3c88129317e663a14f4ba55a3c0c3cc25e9ec2cd4c497276567e4ecfed8a9ab6 targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/rotation_can_bus.raw +8cc4c98a9615a420d40104ae51d4f1f50b719a9cfec4be8d1b65b4b704517def targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/shift.raw +0c9b733c69c569bc02ede7f3684781ef7a0d7b4e881889f8af87ad449374ba41 targets/QCS8550/FP16/code/python/docs/BOARD_ENVIRONMENT.md +4f1d35e184f45b0ccdc9ae3484caa5236d2907a6accef0b0d8fee768de2476a4 targets/QCS8550/FP16/code/python/portable_numpy_nmsfreecoder.py +0026e5f8b3d5d9951f788ea52f374dd55568e18fa4654a647a3474751859dc5f targets/QCS8550/FP16/code/python/run_demo.py +319094897ee71ede6b7892825d9e6e38f5226361cc998d5e8a6f9fd27d6b760a targets/QCS8550/FP16/code/python/run_test.py +eb3a441d362e7c1e2eaee9e3c2b9552232d1e8a36af41d14aefbcf1a4af3b9e3 targets/QCS8550/FP16/code/python/utils.py +ec47d6d7bd68607b805a7b16162eaa012988bef8d845e62e62711be1730e1c28 targets/QCS8550/FP16/composite.json +9a7a5137a2ed3072cdc2c567822a2316b9501a591ffa204b17568f6c80d8844e targets/QCS8550/FP16/models/QCS8550/FP16/backbone_context.bin.aidem +f6888090751bf1ff6ecbaef966b3b1a6c97793784f921001b493893a8ce6196c targets/QCS8550/FP16/models/QCS8550/FP16/decoder_context.bin.aidem +eed541fa4138a177432e2290b4131799d88534005789bef9962e2bbb9e008186 targets/QCS8550/FP16/models/QCS8550/FP16/scene_start_encoder_context.bin.aidem +25f0063fc7d1b125522d3ac9c53b26a9e270d504fc6c908ae5737ec78ad65b94 targets/QCS8550/FP16/models/QCS8550/FP16/temporal_encoder_context.bin.aidem diff --git a/targets/QCS8550/FP16/calibration/sample_000.json b/targets/QCS8550/FP16/calibration/sample_000.json new file mode 100644 index 0000000000000000000000000000000000000000..7deabdd81e61e8bdb9c0db78b5aa4083f69ac4d6 --- /dev/null +++ b/targets/QCS8550/FP16/calibration/sample_000.json @@ -0,0 +1,221 @@ +{ + "cameraOrder": [ + "front", + "front_right", + "front_left", + "back", + "back_left", + "back_right" + ], + "canBus": [ + [ + 0.0, + 0.0, + 0.0, + -0.96875, + -0.96875, + -0.96875, + -0.96875, + -0.60693359375, + -0.07635498046875, + 9.875, + -0.021087646484375, + -0.0124359130859375, + -0.0230712890625, + 8.5625, + 0.0, + 0.0, + 5.78125, + 0.0 + ] + ], + "frame": 0, + "imageSizes": [ + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ] + ], + "lidar2img": [ + [ + [ + [ + 621.5, + 420.25, + 16.375, + -177.125 + ], + [ + -9.1015625, + 268.5, + -613.0, + -322.25 + ], + [ + -0.0117034912109375, + 0.99853515625, + 0.05401611328125, + -0.42529296875 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 682.5, + -309.75, + -20.171875, + -230.875 + ], + [ + 189.75, + 160.125, + -620.0, + -346.25 + ], + [ + 0.84326171875, + 0.5361328125, + 0.03216552734375, + -0.6103515625 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 16.1875, + 751.5, + 38.8125, + -151.25 + ], + [ + -194.625, + 160.25, + -618.5, + -339.75 + ], + [ + -0.8232421875, + 0.56591796875, + 0.041229248046875, + -0.52978515625 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + -402.0, + -425.25, + -13.21875, + -435.5 + ], + [ + -5.41015625, + -222.625, + -407.5, + -354.25 + ], + [ + -0.008331298828125, + -0.9990234375, + -0.039093017578125, + -1.0166015625 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + -593.5, + 461.75, + 26.625, + -312.75 + ], + [ + -231.375, + -51.28125, + -626.0, + -281.0 + ], + [ + -0.94775390625, + -0.319580078125, + 0.003170013427734375, + -0.4326171875 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 142.625, + -734.5, + -29.78125, + -136.25 + ], + [ + 222.375, + -61.40625, + -625.0, + -294.0 + ], + [ + 0.923828125, + -0.38232421875, + -0.0037097930908203125, + -0.464599609375 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ] + ] + ], + "sceneId": "fcbccedd61424f1b85dcbf8f897f9754", + "version": 1 +} diff --git a/targets/QCS8550/FP16/calibration/sample_001.json b/targets/QCS8550/FP16/calibration/sample_001.json new file mode 100644 index 0000000000000000000000000000000000000000..8b5ef31d92301444914c201fcdeed8d56b0a240d --- /dev/null +++ b/targets/QCS8550/FP16/calibration/sample_001.json @@ -0,0 +1,249 @@ +{ + "cameraOrder": [ + "front", + "front_right", + "front_left", + "back", + "back_left", + "back_right" + ], + "canBus": [ + [ + 3.705078125, + -2.103515625, + 0.0, + -0.96630859375, + -0.96630859375, + -0.96630859375, + -0.96630859375, + -0.59765625, + -0.323486328125, + 9.6328125, + 0.01068878173828125, + -0.01297760009765625, + -0.0445556640625, + 8.3515625, + 0.0, + 0.0, + 5.76171875, + -1.03515625 + ] + ], + "frame": 1, + "imageSizes": [ + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ] + ], + "lidar2img": [ + [ + [ + [ + 621.5, + 420.5, + 17.203125, + -180.25 + ], + [ + -8.609375, + 268.75, + -612.5, + -324.5 + ], + [ + -0.012176513671875, + 0.99853515625, + 0.05450439453125, + -0.436279296875 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 682.5, + -309.5, + -19.703125, + -226.75 + ], + [ + 190.25, + 160.5, + -619.5, + -346.5 + ], + [ + 0.84326171875, + 0.53662109375, + 0.033172607421875, + -0.61328125 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 15.640625, + 751.5, + 39.21875, + -161.0 + ], + [ + -194.125, + 160.375, + -619.0, + -341.5 + ], + [ + -0.82373046875, + 0.5654296875, + 0.040618896484375, + -0.53955078125 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + -402.0, + -425.5, + -13.4296875, + -435.5 + ], + [ + -5.23046875, + -222.625, + -407.5, + -354.0 + ], + [ + -0.00823211669921875, + -0.9990234375, + -0.039215087890625, + -1.015625 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + -593.5, + 461.75, + 26.625, + -310.75 + ], + [ + -231.25, + -51.21875, + -626.5, + -281.25 + ], + [ + -0.94775390625, + -0.3193359375, + 0.0030879974365234375, + -0.43359375 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 142.75, + -734.5, + -29.859375, + -130.75 + ], + [ + 222.875, + -61.21875, + -625.0, + -293.0 + ], + [ + 0.92431640625, + -0.382080078125, + -0.0031280517578125, + -0.46044921875 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ] + ] + ], + "rotationCanBus": [ + [ + 3.7056884765625, + -2.103759765625, + 0.0, + -0.9663883447647095, + -0.003500749357044697, + -0.008033331483602524, + 0.25693726539611816, + -0.5976930856704712, + -0.32353678345680237, + 9.631071090698242, + 0.010688905604183674, + -0.01298072375357151, + -0.04455821216106415, + 8.349747657775879, + 0.0, + 0.0, + 5.763484477996826, + -1.035308837890625 + ] + ], + "sceneId": "fcbccedd61424f1b85dcbf8f897f9754", + "shift": [ + [ + -0.0001398324966430664, + 0.0416259765625 + ] + ], + "version": 1 +} diff --git a/targets/QCS8550/FP16/calibration/sample_002.json b/targets/QCS8550/FP16/calibration/sample_002.json new file mode 100644 index 0000000000000000000000000000000000000000..a26f9de9edd7005b3ebe88d5867fbae96590d2e1 --- /dev/null +++ b/targets/QCS8550/FP16/calibration/sample_002.json @@ -0,0 +1,249 @@ +{ + "cameraOrder": [ + "front", + "front_right", + "front_left", + "back", + "back_left", + "back_right" + ], + "canBus": [ + [ + 3.62890625, + -2.171875, + 0.0, + -0.962890625, + -0.962890625, + -0.962890625, + -0.962890625, + -0.35498046875, + -0.417236328125, + 9.7421875, + 0.00868988037109375, + 0.017425537109375, + -0.05926513671875, + 8.234375, + 0.0, + 0.0, + 5.734375, + -1.57421875 + ] + ], + "frame": 2, + "imageSizes": [ + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ] + ], + "lidar2img": [ + [ + [ + [ + 621.0, + 420.75, + 17.234375, + -177.625 + ], + [ + -8.625, + 268.5, + -612.5, + -322.75 + ], + [ + -0.0125732421875, + 0.99853515625, + 0.054351806640625, + -0.4287109375 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 682.5, + -309.25, + -19.671875, + -229.625 + ], + [ + 190.25, + 160.5, + -619.5, + -345.5 + ], + [ + 0.84326171875, + 0.53662109375, + 0.033203125, + -0.60986328125 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 15.3203125, + 751.5, + 39.03125, + -156.0 + ], + [ + -194.0, + 160.125, + -619.0, + -340.5 + ], + [ + -0.82421875, + 0.56494140625, + 0.040283203125, + -0.53515625 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + -402.0, + -425.5, + -13.5234375, + -436.5 + ], + [ + -5.203125, + -222.5, + -407.5, + -354.75 + ], + [ + -0.00809478759765625, + -0.9990234375, + -0.039459228515625, + -1.0185546875 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + -593.5, + 461.75, + 26.640625, + -311.0 + ], + [ + -231.25, + -51.1875, + -626.5, + -281.0 + ], + [ + -0.94775390625, + -0.3193359375, + 0.003086090087890625, + -0.43310546875 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 142.875, + -734.5, + -30.015625, + -136.625 + ], + [ + 222.875, + -61.03125, + -625.0, + -293.5 + ], + [ + 0.92431640625, + -0.3818359375, + -0.003200531005859375, + -0.4638671875 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ] + ] + ], + "rotationCanBus": [ + [ + 3.628173828125, + -2.1719970703125, + 0.0, + -0.9627518057823181, + -0.007396822329610586, + -0.00615654094144702, + 0.2702152729034424, + -0.3548637330532074, + -0.417147159576416, + 9.745100975036621, + 0.008687585592269897, + 0.017426401376724243, + -0.059267766773700714, + 8.235058784484863, + 0.0, + 0.0, + 5.7360124588012695, + -1.574005126953125 + ] + ], + "sceneId": "fcbccedd61424f1b85dcbf8f897f9754", + "shift": [ + [ + -0.0003199577331542969, + 0.041290283203125 + ] + ], + "version": 1 +} diff --git a/targets/QCS8550/FP16/calibration/sample_003.json b/targets/QCS8550/FP16/calibration/sample_003.json new file mode 100644 index 0000000000000000000000000000000000000000..ad58455ab442a980f81af173c10297d7719feb80 --- /dev/null +++ b/targets/QCS8550/FP16/calibration/sample_003.json @@ -0,0 +1,249 @@ +{ + "cameraOrder": [ + "front", + "front_right", + "front_left", + "back", + "back_left", + "back_right" + ], + "canBus": [ + [ + 3.525390625, + -2.236328125, + 0.0, + -0.95849609375, + -0.95849609375, + -0.95849609375, + -0.95849609375, + -0.257080078125, + -0.45556640625, + 9.4375, + 0.00681304931640625, + 0.01085662841796875, + -0.058349609375, + 8.1875, + 0.0, + 0.0, + 5.70703125, + -1.673828125 + ] + ], + "frame": 3, + "imageSizes": [ + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ], + [ + 900, + 1600 + ] + ], + "lidar2img": [ + [ + [ + [ + 621.0, + 421.25, + 17.484375, + -179.375 + ], + [ + -8.7421875, + 268.75, + -612.5, + -326.75 + ], + [ + -0.01325225830078125, + 0.99853515625, + 0.054779052734375, + -0.435546875 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 683.0, + -308.75, + -19.71875, + -226.75 + ], + [ + 190.125, + 160.875, + -619.5, + -348.25 + ], + [ + 0.8427734375, + 0.537109375, + 0.033447265625, + -0.6123046875 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 14.7265625, + 751.5, + 39.4375, + -161.125 + ], + [ + -194.125, + 160.375, + -619.0, + -344.75 + ], + [ + -0.82470703125, + 0.564453125, + 0.04046630859375, + -0.54052734375 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + -401.75, + -425.5, + -13.515625, + -435.0 + ], + [ + -5.16015625, + -222.5, + -407.5, + -354.25 + ], + [ + -0.007904052734375, + -0.9990234375, + -0.0394287109375, + -1.0146484375 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + -593.5, + 461.5, + 26.625, + -312.0 + ], + [ + -231.25, + -51.25, + -626.5, + -281.0 + ], + [ + -0.94775390625, + -0.319580078125, + 0.0030803680419921875, + -0.433349609375 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + [ + [ + 143.25, + -734.5, + -30.078125, + -132.375 + ], + [ + 222.875, + -60.875, + -625.0, + -294.25 + ], + [ + 0.92431640625, + -0.381591796875, + -0.003204345703125, + -0.4609375 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ] + ] + ], + "rotationCanBus": [ + [ + 3.52557373046875, + -2.235595703125, + 0.0, + -0.9586126804351807, + -0.00886558648198843, + -0.013591880910098553, + 0.2842506468296051, + -0.25705933570861816, + -0.45562514662742615, + 9.434834480285645, + 0.006814147345721722, + 0.010853799059987068, + -0.05835989862680435, + 8.190821647644043, + 0.0, + 0.0, + 5.706801891326904, + -1.67364501953125 + ] + ], + "sceneId": "fcbccedd61424f1b85dcbf8f897f9754", + "shift": [ + [ + -0.0004591941833496094, + 0.040771484375 + ] + ], + "version": 1 +} diff --git a/targets/QCS8550/FP16/code/.gitkeep b/targets/QCS8550/FP16/code/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/targets/QCS8550/FP16/code/README.md b/targets/QCS8550/FP16/code/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7c446be54ba83d88744d2ebbc55414f99a5b996e --- /dev/null +++ b/targets/QCS8550/FP16/code/README.md @@ -0,0 +1,150 @@ +## Model Information + +### Source model +- Input shape: [[1,6,3,480,800]] +- Number of parameters: 33.7M +- Model size: 128.5 FP32 +- Output shape: boxes [[300,9]], scores [[300]], labels [[300]] + +Source model repository: [BEVFormer](https://github.com/fundamentalvision/bevformer) + +## Inference with AidLite SDK + +### SDK installation +Model Farm uses AidLite SDK as the model inference SDK. For details, please refer to the [AidLite Developer Documentation](https://docs.aidlux.com/software/ai-sdk/aidlite_guide) + +```bash +sudo aid-pkg update +sudo aid-pkg install aidlite-sdk +sudo aid-pkg install aidlite-{QNN VERSION} +``` + +Verify AidLite SDK: + +```bash +python3 -c "import aidlite ; print(aidlite.get_library_version())" +python3 -c "import aidlite ; print(aidlite.get_py_library_version())" +``` + +### Environment Dependencies + +Except for AidLite SDK, this BEVFormer demo also requires the following board-side Python environment. + +Verified environment: + +```bash +Python 3.10.12 +numpy 1.26.4 +opencv-python / cv2 4.13.0 +Pillow / PIL 10.4.0 +``` + +Required Python packages: + +```bash +numpy +opencv-python +Pillow +``` + +Quick check: + +```bash +python3 -c "import numpy, cv2; from PIL import Image; print('python env ok')" +``` + +The demo supports model directories for QCS8550, IQ9, QCS8625, and IQ8. A normal host or development container can only run dry-run package checks and cannot execute real QNN inference without AidLite. + +## Model Directory Rule + +Model SHA verification is always enabled. The platform is inferred from the model path, so no extra SHA option is required. + +Supported model directories: + +```bash +./models/QCS8550/FP16 +./models/IQ9/FP16 +./models/QCS8625/FP16 +./models/IQ8/FP16 +``` + +Platform mapping: + +```text +QCS8550 -> QCS8550 model SHA table +IQ9 -> QCS8550 model SHA table +QCS8625 -> QCS8625 model SHA table +IQ8 -> QCS8625 model SHA table +``` + +Each platform directory must contain all four context files: + +```bash +backbone_context.bin.aidem +scene_start_encoder_context.bin.aidem +temporal_encoder_context.bin.aidem +decoder_context.bin.aidem +``` + +If all four `--*_model` arguments are provided, the model paths decide the platform. All four paths must belong to the same platform directory. If model arguments are omitted, the script uses the first complete directory found in this order: + +```text +QCS8550, IQ9, QCS8625, IQ8 +``` + +## Run Demo + +Basic command: + +```bash +cd /home/aidlux/bevformer_delivery_demo_2026_07_08 +python3 code/python/run_test.py --invoke_nums 4 --output_dir ./outputs/final_sample4 +``` + +Full command with explicit model paths and dataset manifest: + +```bash +python3 code/python/run_test.py \ + --backbone_model ./models/QCS8625/FP16/backbone_context.bin.aidem \ + --scene_start_encoder_model ./models/QCS8625/FP16/scene_start_encoder_context.bin.aidem \ + --encoder_model ./models/QCS8625/FP16/temporal_encoder_context.bin.aidem \ + --decoder_model ./models/QCS8625/FP16/decoder_context.bin.aidem \ + --asset_manifest ./code/python/datasets/sample4/asset_manifest.json \ + --invoke_nums 4 \ + --output_dir ./outputs/final_sample4_qcs8625 +``` + +The basic command auto-discovers the first complete model directory under `models//FP16`. For QCS8550, IQ9, or IQ8, replace `QCS8625` in the full command with `QCS8550`, `IQ9`, or `IQ8`. The four model paths must use the same platform directory. + +Dry-run package check: + +```bash +python3 code/python/run_test.py --dry_run --check_raw_assets +``` + +The demo input is specified by: + +```bash +./code/python/datasets/sample4/asset_manifest.json +``` + +The manifest describes four continuous BEVFormer frames. Each frame contains six raw camera JPG images and the required calibration / temporal auxiliary tensors. The demo starts from original six-camera images, not preprocessed image tensors. + +Frame execution order: + +```bash +frame 000: scene-start encoder +frame 001: temporal encoder +frame 002: temporal encoder +frame 003: temporal encoder +``` + +Main output files include: + +```bash +bevformer_demo_summary.json +run.log +frameXXX_final_coordinates.npz +frameXXX_camera_grid.png +sample4_camera_grid.gif +``` diff --git a/targets/QCS8550/FP16/code/python/bevformer.py b/targets/QCS8550/FP16/code/python/bevformer.py new file mode 100644 index 0000000000000000000000000000000000000000..7411316c3cdb9d3c8faa7a7a04865c7844d448ea --- /dev/null +++ b/targets/QCS8550/FP16/code/python/bevformer.py @@ -0,0 +1,611 @@ +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +import numpy as np + +import portable_numpy_nmsfreecoder as portable_nms +from camera_grid_visualization import save_camera_grid_gif, save_camera_grid_summary, save_camera_grid_visualization +from utils import ( + EXPECTED_TENSORS, + SHAPES, + as_encoder_img_feat, + elapsed_ms, + load_backbone_images, + load_json, + load_record, + normalize_rc, + rotate_prev_bev, + save_final_coordinates, + save_raw_outputs, + sha256_file, + stats, +) + + +QCS8550_SHA256 = { + "backbone": "9a7a5137a2ed3072cdc2c567822a2316b9501a591ffa204b17568f6c80d8844e", + "encoder_temporal": "25f0063fc7d1b125522d3ac9c53b26a9e270d504fc6c908ae5737ec78ad65b94", + "encoder_scene_start": "eed541fa4138a177432e2290b4131799d88534005789bef9962e2bbb9e008186", + "decoder": "f6888090751bf1ff6ecbaef966b3b1a6c97793784f921001b493893a8ce6196c", +} + +IQ9_SHA256 = { + "backbone": "54c042b09eff47d99998762a375fe606bc3a83350ebf1085399384f50fe2dbfb", + "encoder_temporal": "b376fe6112794278a4b3f787b34e387d62cd81de00312021ca927448c4e9c253", + "encoder_scene_start": "8b0987fe7a21a79a4bc5975f3732b7d9c60611fadc813a96e438dfc85a59fb2e", + "decoder": "60952eaa1d35c648d5992a4f8ef2563410b8b6fe68e8ad6095f1231921ea0ff5", +} + +QCS8625_SHA256 = { + "backbone": "552bb2f456638c19bfcd88d79e5c15a671fddca21077332c6e0cb1d1cd2641de", + "encoder_temporal": "046652a264dce3728b11433b1a3378e6c4b894e1286d007d940daec4ec5a2608", + "encoder_scene_start": "7071b3937d79191412db50688136f88f70c917a8bff6c0e8af649157555be592", + "decoder": "be322fead0610e0321ce165cc0c1d2686768b6c6e1b3241c8ef89f45f6d2bbc5", +} + +IQ8_SHA256 = { + "backbone": "e7f75ca4ca7fda12ecdfbb14694a906165b70ecdc8e605b54174d82a042f25b1", + "encoder_temporal": "f79e2c208fe63eec8f30bf289310dcc429cf6b87a05dd3397aeda6d08aabaf74", + "encoder_scene_start": "62c6b62139525d238c447d65f47004f65bd9e2cd88beae5b580bdd7d80687f81", + "decoder": "814534559bf5ffd00f4316302b7804434e19a2420117ec9b5f46fe5dbce887f1", +} + +MODEL_SHA256 = { + "QCS8550": QCS8550_SHA256, + "IQ9": IQ9_SHA256, + "QCS8625": QCS8625_SHA256, + "IQ8": IQ8_SHA256, +} + + +MODEL_NAMES = ( + "backbone", + "encoder_temporal", + "encoder_scene_start", + "decoder", +) + +NUSCENES_CLASSES = ( + "car", + "truck", + "construction_vehicle", + "bus", + "trailer", + "barrier", + "motorcycle", + "bicycle", + "pedestrian", + "traffic_cone", +) + + +class BevFormerModel: + def __init__( + self, + backbone_model: str, + encoder_temporal_model: str, + encoder_scene_start_model: str, + decoder_model: str, + model_type: str = "QNN248", + expected_sha256: dict[str, str] | None = None, + ): + try: + import aidlite + except ModuleNotFoundError as exc: + raise RuntimeError( + "AidLite Python runtime is not available in this environment. " + "Use --dry_run to demonstrate the package structure in a normal container, " + "or run without --dry_run on the board / Container B where " + "`python3 -c \"import aidlite\"` succeeds." + ) from exc + + if model_type.upper() != "QNN248": + raise ValueError("This demo is pinned to QNN248 contexts") + if ( + int(aidlite.FrameworkType.TYPE_QNN248), + int(aidlite.ImplementType.TYPE_LOCAL), + int(aidlite.AccelerateType.TYPE_DSP), + ) != (110, 3, 3): + raise RuntimeError("AidLite enum contract mismatch") + + self.aidlite = aidlite + model_paths = { + "backbone": backbone_model, + "encoder_temporal": encoder_temporal_model, + "encoder_scene_start": encoder_scene_start_model, + "decoder": decoder_model, + } + self.model_platform = self.resolve_model_platform(model_paths) + self.expected_sha256 = expected_sha256 or MODEL_SHA256[self.model_platform] + self.interpreters: dict[str, Any] = {} + self.model_records: dict[str, Any] = {} + self.model_load_timing_ms: dict[str, float] = {} + model_load_start = time.perf_counter_ns() + + for name in MODEL_NAMES: + path = model_paths[name] + load_start = time.perf_counter_ns() + interpreter, record = self._create_loaded_interpreter(name, str(path)) + self.model_load_timing_ms[name] = elapsed_ms(load_start) + self.interpreters[name] = interpreter + self.model_records[name] = record + + self.model_load_timing_ms["total"] = elapsed_ms(model_load_start) + + def __del__(self): + for interpreter in reversed(list(getattr(self, "interpreters", {}).values())): + for method_name in ("destroy", "destory"): + if hasattr(interpreter, method_name): + try: + getattr(interpreter, method_name)() + except Exception: + pass + break + + def _create_model(self, model_path: str) -> Any: + try: + return self.aidlite.Model.create_instance(model_path=model_path) + except TypeError: + return self.aidlite.Model.create_instance(model_path) + + def _build_interpreter(self, model: Any, config: Any) -> Any: + for method_name in ("build_interpreter_from_model_and_config", "build_interpretper_from_model_and_config"): + if hasattr(self.aidlite.InterpreterBuilder, method_name): + method = getattr(self.aidlite.InterpreterBuilder, method_name) + try: + return method(model=model, config=config) + except TypeError: + return method(model, config) + raise RuntimeError("No supported AidLite InterpreterBuilder method") + + @staticmethod + def infer_model_platform(model_path: str | Path) -> str: + parts = [part.upper() for part in Path(model_path).expanduser().parts] + matches = [platform for platform in MODEL_SHA256 if platform.upper() in parts] + if len(matches) != 1: + supported = ", ".join(MODEL_SHA256) + raise RuntimeError(f"Cannot infer model platform from path={model_path}. Expected one of: {supported}") + return matches[0] + + @classmethod + def resolve_model_platform(cls, model_paths: dict[str, str | Path]) -> str: + platforms = {name: cls.infer_model_platform(path) for name, path in model_paths.items()} + unique = set(platforms.values()) + if len(unique) != 1: + raise RuntimeError(f"Model platform mismatch: {platforms}") + return next(iter(unique)) + + @staticmethod + def _flatten_tensor_info(groups: Any) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + if groups is None: + return records + for graph_index, group in enumerate(groups): + try: + tensors = list(group) + except TypeError: + tensors = [group] + for tensor_index, info in enumerate(tensors): + records.append({ + "graph_index": graph_index, + "tensor_index": tensor_index, + "name": str(getattr(info, "name", "")), + "element_count": int(getattr(info, "element_count", -1)), + "shape": [int(v) for v in getattr(info, "shape", [])], + "element_type": str(getattr(info, "element_type", "")), + }) + return records + + def _create_loaded_interpreter(self, name: str, model_path: str) -> tuple[Any, dict[str, Any]]: + path = Path(model_path).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(path) + actual_sha = sha256_file(path) + expected_sha = self.expected_sha256[name] + if actual_sha != expected_sha: + raise RuntimeError( + f"{name} context SHA mismatch for {self.model_platform}: " + f"expected={expected_sha} actual={actual_sha}" + ) + + model = self._create_model(str(path)) + config = self.aidlite.Config.create_instance() + if model is None or config is None: + raise RuntimeError(f"{name}: Model/Config creation failed") + config.framework_type = self.aidlite.FrameworkType.TYPE_QNN248 + config.implement_type = self.aidlite.ImplementType.TYPE_LOCAL + config.accelerate_type = self.aidlite.AccelerateType.TYPE_DSP + config.qnn_shared_buffer = 0 + + interpreter = self._build_interpreter(model, config) + if interpreter is None: + raise RuntimeError(f"{name}: interpreter creation failed") + init_rc = normalize_rc(interpreter.init()) + load_rc = normalize_rc(interpreter.load_model()) + if init_rc != 0 or load_rc != 0: + raise RuntimeError(f"{name}: init/load failed init={init_rc} load={load_rc}") + + inputs = self._flatten_tensor_info(interpreter.get_input_tensor_info()) + outputs = self._flatten_tensor_info(interpreter.get_output_tensor_info()) + actual_inputs = {item["name"]: item["element_count"] for item in inputs} + actual_outputs = {item["name"]: item["element_count"] for item in outputs} + expected = EXPECTED_TENSORS[name] + if actual_inputs != expected["inputs"] or actual_outputs != expected["outputs"]: + raise RuntimeError(f"{name}: tensor contract mismatch inputs={actual_inputs} outputs={actual_outputs}") + + return interpreter, { + "name": name, + "platform": self.model_platform, + "path": str(path), + "sha256": actual_sha, + "inputs": inputs, + "outputs": outputs, + } + + def _set_input(self, interpreter: Any, name: str, value: np.ndarray) -> float: + tensor = np.ascontiguousarray(value, dtype=np.float32) + start = time.perf_counter_ns() + rc = normalize_rc(interpreter.set_input_tensor(in_tensor_tag=name, input_data=tensor)) + duration = elapsed_ms(start) + if rc != 0: + raise RuntimeError(f"set_input_tensor failed name={name} rc={rc}") + return duration + + def _invoke(self, interpreter: Any, name: str) -> float: + start = time.perf_counter_ns() + rc = normalize_rc(interpreter.invoke()) + duration = elapsed_ms(start) + if rc != 0: + raise RuntimeError(f"{name}: invoke failed rc={rc}") + return duration + + def _get_output(self, interpreter: Any, name: str, shape: tuple[int, ...]) -> tuple[np.ndarray, float]: + start = time.perf_counter_ns() + value = interpreter.get_output_tensor(out_tensor_tag=name) + duration = elapsed_ms(start) + if value is None: + raise RuntimeError(f"get_output_tensor returned None name={name}") + array = np.asarray(value, dtype=np.float32).reshape(shape) + if not np.isfinite(array).all(): + raise RuntimeError(f"{name}: non-finite output") + return np.ascontiguousarray(array, dtype=np.float32), duration + + def run_frame( + self, + frame_index: int, + frame_manifest: dict[str, Any], + repo_root: str | Path, + previous_frame_bev_embed: np.ndarray | None, + check_image_sha: bool = False, + preprocess_workers: int = 6, + ) -> tuple[np.ndarray, np.ndarray, np.ndarray, dict[str, Any]]: + """Run one BEVFormer frame on the board. + + The QNN context tensor names still follow the exported model contract + such as ``can_bus``, ``lidar2img`` and ``prev_bev``. The Python + variable names below describe their deployment meaning more explicitly: + + - scene-start frame: use the scene-start encoder; no previous BEV state. + - temporal frame: rotate the previous frame BEV state, then feed it as + the current frame ``prev_bev`` tensor. + """ + + frame_start = time.perf_counter_ns() + is_scene_start_frame = bool(frame_manifest.get("is_scene_start", False)) + asset_records = frame_manifest["assets"] + timing: dict[str, float] = {} + + preprocess_start = time.perf_counter_ns() + padded_camera_tensor, image_preprocess = load_backbone_images( + asset_records, + repo_root, + check_image_sha=check_image_sha, + preprocess_workers=preprocess_workers, + ) + timing["image_preprocess_ms"] = elapsed_ms(preprocess_start) + + ego_motion_can_bus = load_record(asset_records["can_bus"], repo_root).astype(np.float32) + camera_lidar2img_matrices = load_record(asset_records["lidar2img"], repo_root).astype(np.float32) + + timing["backbone_set_input_ms"] = self._set_input( + self.interpreters["backbone"], + "images", + padded_camera_tensor, + ) + timing["backbone_invoke_ms"] = self._invoke(self.interpreters["backbone"], "backbone") + backbone_image_features, timing["backbone_get_output_ms"] = self._get_output( + self.interpreters["backbone"], "img_feat", SHAPES["img_feat"] + ) + encoder_image_features = as_encoder_img_feat(backbone_image_features) + + if is_scene_start_frame: + selected_encoder_name = "encoder_scene_start" + selected_encoder_route = "scene_start_reset_prev_bev" + selected_encoder = self.interpreters[selected_encoder_name] + timing["encoder_set_input_ms"] = 0.0 + scene_start_inputs = ( + ("can_bus", ego_motion_can_bus), + ("img_feat", encoder_image_features), + ("lidar2img", camera_lidar2img_matrices), + ) + for tensor_name, tensor_value in scene_start_inputs: + timing["encoder_set_input_ms"] += self._set_input(selected_encoder, tensor_name, tensor_value) + else: + if previous_frame_bev_embed is None: + raise RuntimeError(f"frame{frame_index:03d}: previous BEV state is missing for temporal frame") + + ego_motion_shift = load_record(asset_records["shift"], repo_root).astype(np.float32) + ego_rotation_can_bus = load_record(asset_records["rotation_can_bus"], repo_root).astype(np.float32) + rotate_start = time.perf_counter_ns() + rotated_previous_bev_embed = rotate_prev_bev(previous_frame_bev_embed, ego_rotation_can_bus) + timing["prev_bev_rotate_ms"] = elapsed_ms(rotate_start) + + selected_encoder_name = "encoder_temporal" + selected_encoder_route = "temporal_reuse_previous_bev" + selected_encoder = self.interpreters[selected_encoder_name] + timing["encoder_set_input_ms"] = 0.0 + temporal_inputs = ( + ("can_bus", ego_motion_can_bus), + ("img_feat", encoder_image_features), + ("lidar2img", camera_lidar2img_matrices), + ("shift", ego_motion_shift), + ("prev_bev", rotated_previous_bev_embed), + ) + for tensor_name, tensor_value in temporal_inputs: + timing["encoder_set_input_ms"] += self._set_input(selected_encoder, tensor_name, tensor_value) + + timing["encoder_invoke_ms"] = self._invoke(selected_encoder, selected_encoder_name) + current_frame_bev_embed, timing["encoder_get_output_ms"] = self._get_output( + selected_encoder, + "bev_embed", + SHAPES["bev"], + ) + + decoder = self.interpreters["decoder"] + timing["decoder_set_input_ms"] = self._set_input(decoder, "bev_embed", current_frame_bev_embed) + timing["decoder_invoke_ms"] = self._invoke(decoder, "decoder") + decoder_cls_scores, timing["decoder_get_cls_ms"] = self._get_output(decoder, "cls_scores", SHAPES["decoder"]) + decoder_bbox_preds, timing["decoder_get_bbox_ms"] = self._get_output(decoder, "bbox_preds", SHAPES["decoder"]) + + timing["qnn_invoke_ms"] = ( + timing["backbone_invoke_ms"] + + timing["encoder_invoke_ms"] + + timing["decoder_invoke_ms"] + ) + timing["frame_total_ms"] = elapsed_ms(frame_start) + frame_result = { + "frame_index": int(frame_index), + "sample_token": frame_manifest.get("sample_token"), + "frame_type": "scene_start" if is_scene_start_frame else "temporal", + "is_scene_start": is_scene_start_frame, + "encoder": selected_encoder_name, + "encoder_route": selected_encoder_route, + "image_preprocess": image_preprocess, + "timing_ms": timing, + "status": "PASS", + } + return current_frame_bev_embed, decoder_cls_scores, decoder_bbox_preds, frame_result + + def run_manifest( + self, + manifest_path: str | Path, + repo_root: str | Path, + output_dir: str | Path, + nms_contract_path: str | Path, + frame_start: int = 0, + frame_count: int | None = None, + save_all_raw: bool = False, + visualize: bool = True, + vis_score_thr: float = 0.0, + vis_max_boxes: int = 80, + check_image_sha: bool = False, + preprocess_workers: int = 6, + ) -> dict[str, Any]: + run_manifest_start = time.perf_counter_ns() + manifest_load_start = time.perf_counter_ns() + manifest = load_json(manifest_path) + nms_contract = load_json(nms_contract_path) + manifest_load_ms = elapsed_ms(manifest_load_start) + total_frames = int(manifest.get("total_frames", len(manifest["frames"]))) + end = total_frames if frame_count is None else min(total_frames, frame_start + frame_count) + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + previous_frame_bev_embed: np.ndarray | None = None + frames: dict[str, Any] = {} + scene_start_count = 0 + temporal_count = 0 + frame_ms: list[float] = [] + qnn_invoke_ms: list[float] = [] + backbone_invoke_ms: list[float] = [] + encoder_invoke_ms: list[float] = [] + scene_start_encoder_invoke_ms: list[float] = [] + temporal_encoder_invoke_ms: list[float] = [] + decoder_invoke_ms: list[float] = [] + image_preprocess_ms: list[float] = [] + raw_save_ms: list[float] = [] + nms_decode_ms: list[float] = [] + final_coordinates_save_ms: list[float] = [] + postprocess_ms: list[float] = [] + visualization_ms: list[float] = [] + final_outputs: dict[str, Any] | None = None + final_coordinates: dict[str, Any] = {} + visualizations: dict[str, Any] = {} + + for frame_index in range(frame_start, end): + sample = f"sample_{frame_index:03d}" + current_frame_bev_embed, decoder_cls_scores, decoder_bbox_preds, frame_record = self.run_frame( + frame_index, + manifest["frames"][sample], + repo_root, + previous_frame_bev_embed, + check_image_sha=check_image_sha, + preprocess_workers=preprocess_workers, + ) + previous_frame_bev_embed = np.ascontiguousarray(current_frame_bev_embed, dtype=np.float32) + frames[sample] = frame_record + frame_ms.append(frame_record["timing_ms"]["frame_total_ms"]) + qnn_invoke_ms.append(frame_record["timing_ms"]["qnn_invoke_ms"]) + backbone_invoke_ms.append(frame_record["timing_ms"]["backbone_invoke_ms"]) + encoder_time = frame_record["timing_ms"]["encoder_invoke_ms"] + encoder_invoke_ms.append(encoder_time) + if frame_record["encoder"] == "encoder_scene_start": + scene_start_encoder_invoke_ms.append(encoder_time) + elif frame_record["encoder"] == "encoder_temporal": + temporal_encoder_invoke_ms.append(encoder_time) + decoder_invoke_ms.append(frame_record["timing_ms"]["decoder_invoke_ms"]) + image_preprocess_ms.append(frame_record["timing_ms"].get("image_preprocess_ms", 0.0)) + if frame_record["is_scene_start"]: + scene_start_count += 1 + else: + temporal_count += 1 + raw_save_value = 0.0 + if save_all_raw or frame_index == end - 1: + raw_save_start = time.perf_counter_ns() + final_outputs = save_raw_outputs(output_path, frame_index, decoder_cls_scores, decoder_bbox_preds) + raw_save_value = elapsed_ms(raw_save_start) + raw_save_ms.append(raw_save_value) + + nms_start = time.perf_counter_ns() + boxes, scores, labels = portable_nms.decode_numpy_nmsfreecoder( + decoder_cls_scores, + decoder_bbox_preds, + nms_contract, + ) + nms_value = elapsed_ms(nms_start) + nms_decode_ms.append(nms_value) + + final_save_start = time.perf_counter_ns() + final_coordinates[f"frame{frame_index:03d}"] = save_final_coordinates( + output_path, + frame_index, + boxes, + scores, + labels, + ) + final_save_value = elapsed_ms(final_save_start) + final_coordinates_save_ms.append(final_save_value) + postprocess_ms.append(raw_save_value + nms_value + final_save_value) + + visualization_value = 0.0 + if visualize: + visualization_start = time.perf_counter_ns() + visualizations[f"frame{frame_index:03d}"] = save_camera_grid_visualization( + output_path, + frame_index, + manifest["frames"][sample], + repo_root, + boxes, + scores, + labels, + score_thr=vis_score_thr, + max_boxes=vis_max_boxes, + result_dir=output_path, + ) + visualization_value = elapsed_ms(visualization_start) + visualization_ms.append(visualization_value) + frame_record["timing_ms"]["raw_save_ms"] = raw_save_value + frame_record["timing_ms"]["nms_decode_ms"] = nms_value + frame_record["timing_ms"]["final_coordinates_save_ms"] = final_save_value + frame_record["timing_ms"]["postprocess_ms"] = raw_save_value + nms_value + final_save_value + frame_record["timing_ms"]["visualization_ms"] = visualization_value + top_detections = [] + for det_index in range(min(5, len(scores))): + label_id = int(labels[det_index]) + class_name = NUSCENES_CLASSES[label_id] if label_id < len(NUSCENES_CLASSES) else str(label_id) + top_detections.append({ + "box": [float(value) for value in boxes[det_index].tolist()], + "score": float(scores[det_index]), + "label": label_id, + "class_name": class_name, + }) + frame_record["detections"] = { + "count": int(len(scores)), + "top": top_detections, + } + route_text = ( + "first frame: reset prev_bev state" + if frame_record["encoder_route"] == "scene_start_reset_prev_bev" + else "later frame: previous bev_embed -> prev_bev" + ) + print(f"Frame {frame_index:03d} | QNN={frame_record['timing_ms']['qnn_invoke_ms']:.1f} ms backbone={frame_record['timing_ms']['backbone_invoke_ms']:.1f} encoder={frame_record['timing_ms']['encoder_invoke_ms']:.1f} decoder={frame_record['timing_ms']['decoder_invoke_ms']:.1f}") + + camera_grid_gif = None + camera_grid_summary = None + gif_ms = 0.0 + if visualize and visualizations: + gif_start = time.perf_counter_ns() + ordered_records = [visualizations[key] for key in sorted(visualizations)] + camera_grid_gif = save_camera_grid_gif(ordered_records, output_path) + camera_grid_summary = save_camera_grid_summary(output_path, ordered_records, camera_grid_gif) + gif_ms = elapsed_ms(gif_start) + if camera_grid_gif: + print(f"GIF: {camera_grid_gif['path']}") + + total_with_visualization_ms = elapsed_ms(run_manifest_start) + total_without_visualization_ms = ( + manifest_load_ms + + sum(frame_ms) + + sum(postprocess_ms) + ) + total_visualization_ms = sum(visualization_ms) + gif_ms + + return { + "status": "PASS", + "manifest": str(Path(manifest_path).resolve()), + "nms_contract": str(Path(nms_contract_path).resolve()), + "repo_root": str(Path(repo_root).resolve()), + "frame_range": [int(frame_start), int(end - 1)] if end > frame_start else [], + "completed_frames": int(end - frame_start), + "scene_start_encoder_count": scene_start_count, + "temporal_encoder_count": temporal_count, + "model_platform": self.model_platform, + "models": self.model_records, + "model_load_timing_ms": self.model_load_timing_ms, + "end_to_end_timing_ms": { + "manifest_and_contract_load_ms": manifest_load_ms, + "complete_inference_no_visualization_ms": total_without_visualization_ms, + "visualization_total_ms": total_visualization_ms, + "complete_inference_with_visualization_ms": total_with_visualization_ms, + "camera_grid_gif_ms": gif_ms, + }, + "timing_ms": stats(frame_ms), + "qnn_invoke_ms": stats(qnn_invoke_ms), + "component_invoke_ms": { + "backbone": stats(backbone_invoke_ms), + "encoder": stats(encoder_invoke_ms), + "encoder_scene_start": stats(scene_start_encoder_invoke_ms), + "encoder_temporal": stats(temporal_encoder_invoke_ms), + "decoder": stats(decoder_invoke_ms), + }, + "per_bin_qnn_invoke_ms": { + "backbone_context.bin": stats(backbone_invoke_ms), + "scene_start_encoder_context.bin": stats(scene_start_encoder_invoke_ms), + "temporal_encoder_context.bin": stats(temporal_encoder_invoke_ms), + "decoder_context.bin": stats(decoder_invoke_ms), + }, + "image_preprocess_ms": stats(image_preprocess_ms), + "postprocess_ms": stats(postprocess_ms), + "nms_decode_ms": stats(nms_decode_ms), + "raw_save_ms": stats(raw_save_ms), + "final_coordinates_save_ms": stats(final_coordinates_save_ms), + "visualization_ms": stats(visualization_ms), + "timing_contract": { + "frame_total_ms": "Per-frame inference path: image preprocessing plus model input/output and QNN invoke time; excludes NMS, result saving, and camera-grid visualization rendering.", + "qnn_invoke_ms": "Backbone + selected encoder + decoder invoke time only. Excludes preprocessing, tensor set/get, NMS, result saving, and visualization.", + "per_bin_qnn_invoke_ms": "Pure AidLite/QNN interpreter.invoke() time grouped by delivered .bin context file.", + "image_preprocess_ms": "Six-camera JPG decode, RGB conversion, normalization, resize, CHW conversion, and zero padding.", + "complete_inference_no_visualization_ms": "Manifest load plus all frame inference, NMS, and result saving; excludes PNG/GIF rendering.", + "complete_inference_with_visualization_ms": "Full run_manifest wall time including PNG/GIF rendering; excludes Python process startup and model loading, which are reported separately.", + }, + "frames": frames, + "final_outputs": final_outputs, + "final_coordinates": final_coordinates, + "visualizations": visualizations, + "camera_grid_gif": camera_grid_gif, + "camera_grid_summary": camera_grid_summary, + } + diff --git a/targets/QCS8550/FP16/code/python/camera_grid_visualization.py b/targets/QCS8550/FP16/code/python/camera_grid_visualization.py new file mode 100644 index 0000000000000000000000000000000000000000..f068556d6a5d04034d1c16d9af8e61b79961a6fb --- /dev/null +++ b/targets/QCS8550/FP16/code/python/camera_grid_visualization.py @@ -0,0 +1,309 @@ +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any, Sequence + +import numpy as np +from PIL import Image, ImageDraw, ImageFont + +from utils import load_record, resolve_path, sha256_file + + +CLASS_NAMES = ( + "car", + "truck", + "construction_vehicle", + "bus", + "trailer", + "barrier", + "motorcycle", + "bicycle", + "pedestrian", + "traffic_cone", +) + +MODEL_CAMERA_ORDER = ( + "CAM_FRONT", + "CAM_FRONT_RIGHT", + "CAM_FRONT_LEFT", + "CAM_BACK", + "CAM_BACK_LEFT", + "CAM_BACK_RIGHT", +) + +GRID_CAMERA_ORDER = ( + "CAM_FRONT_LEFT", + "CAM_FRONT", + "CAM_FRONT_RIGHT", + "CAM_BACK_LEFT", + "CAM_BACK", + "CAM_BACK_RIGHT", +) + +EDGES = ( + (0, 1), (1, 2), (2, 3), (3, 0), + (4, 5), (5, 6), (6, 7), (7, 4), + (0, 4), (1, 5), (2, 6), (3, 7), +) + + + +def load_font(size: int = 18): + for path in ( + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + "/usr/share/fonts/truetype/liberation2/LiberationSans-Regular.ttf", + ): + if Path(path).is_file(): + return ImageFont.truetype(path, size) + return ImageFont.load_default() + + +def corners_3d(box: np.ndarray) -> np.ndarray: + x, y, z, w, length, h, yaw = [float(v) for v in box[:7]] + x_c = np.array([length / 2, length / 2, -length / 2, -length / 2, length / 2, length / 2, -length / 2, -length / 2]) + y_c = np.array([w / 2, -w / 2, -w / 2, w / 2, w / 2, -w / 2, -w / 2, w / 2]) + z_c = np.array([h / 2, h / 2, h / 2, h / 2, -h / 2, -h / 2, -h / 2, -h / 2]) + rot = np.array( + [ + [math.cos(yaw), -math.sin(yaw), 0.0], + [math.sin(yaw), math.cos(yaw), 0.0], + [0.0, 0.0, 1.0], + ], + dtype=np.float64, + ) + return (rot @ np.stack([x_c, y_c, z_c], axis=0)).T + np.array([x, y, z], dtype=np.float64) + + +def project_box(box: np.ndarray, lidar2img: np.ndarray, image_size: tuple[int, int]) -> np.ndarray | None: + corners = corners_3d(box) + homo = np.concatenate([corners, np.ones((8, 1), dtype=np.float64)], axis=1) + proj = (lidar2img.astype(np.float64) @ homo.T).T + depth = proj[:, 2] + if np.count_nonzero(depth > 1e-3) < 8: + return None + points = proj[:, :2] / depth[:, None] + width, height = image_size + if not ( + (points[:, 0] >= -100).any() + and (points[:, 0] <= width + 100).any() + and (points[:, 1] >= -100).any() + and (points[:, 1] <= height + 100).any() + ): + return None + return points + + +def draw_projected_box(draw: ImageDraw.ImageDraw, points: np.ndarray, color: tuple[int, int, int], width: int = 3) -> None: + xy = [(float(x), float(y)) for x, y in points] + for i, j in EDGES: + draw.line([xy[i], xy[j]], fill=color, width=width) + + +def draw_label(draw: ImageDraw.ImageDraw, xy: tuple[float, float], text: str, color: tuple[int, int, int], font) -> None: + x = max(0.0, min(float(xy[0]), 790.0)) + y = max(0.0, min(float(xy[1]), 440.0)) + bbox = draw.textbbox((x, y), text, font=font) + draw.rectangle(bbox, fill=(0, 0, 0)) + draw.text((x, y), text, fill=color, font=font) + + +def select_predictions( + boxes: np.ndarray, + scores: np.ndarray, + labels: np.ndarray, + score_thr: float, + max_boxes: int, +) -> list[tuple[np.ndarray, float, str]]: + boxes = np.asarray(boxes, dtype=np.float64) + scores = np.asarray(scores, dtype=np.float64).reshape(-1) + labels = np.asarray(labels, dtype=np.int64).reshape(-1) + order = np.argsort(-scores, kind="stable") + out: list[tuple[np.ndarray, float, str]] = [] + for idx in order: + if float(scores[idx]) < float(score_thr): + continue + label = int(labels[idx]) + name = CLASS_NAMES[label] if 0 <= label < len(CLASS_NAMES) else str(label) + out.append((boxes[idx], float(scores[idx]), name)) + if len(out) >= int(max_boxes): + break + return out + + +def camera_records(frame_assets: dict[str, Any], repo_root: str | Path) -> dict[str, Path]: + record = frame_assets["assets"]["camera_images"] + return {item["name"]: resolve_path(item["path"], repo_root) for item in record["images"]} + + +def load_lidar2img_by_camera(frame_assets: dict[str, Any], repo_root: str | Path) -> dict[str, np.ndarray]: + value = load_record(frame_assets["assets"]["lidar2img"], repo_root).astype(np.float64).reshape(1, 6, 4, 4)[0] + return {name: value[index] for index, name in enumerate(MODEL_CAMERA_ORDER)} + + +def wrap_text(text: str, width: int = 42) -> list[str]: + words = str(text).split() + lines: list[str] = [] + current = "" + for word in words: + trial = word if not current else current + " " + word + if len(trial) <= width: + current = trial + else: + if current: + lines.append(current) + current = word + if current: + lines.append(current) + return lines + + +def add_sidebar(sheet: Image.Image, frame_index: int, branch: str, score_thr: float, max_boxes: int, result_dir: str | Path | None) -> Image.Image: + sidebar_w = 520 + canvas = Image.new("RGB", (sheet.width + sidebar_w, sheet.height), (250, 250, 250)) + canvas.paste(sheet, (0, 0)) + draw = ImageDraw.Draw(canvas) + title_font = load_font(24) + body_font = load_font(18) + small_font = load_font(15) + x0 = sheet.width + 24 + y = 24 + draw.rectangle((sheet.width, 0, canvas.width - 1, canvas.height - 1), fill=(245, 247, 250), outline=(180, 180, 180), width=2) + draw.text((x0, y), "Final Detection Visualization", fill=(0, 0, 0), font=title_font) + y += 46 + lines = [ + f"frame{frame_index:03d} / {branch}", + "Final Model: W8A8 per_channel", + "Role: AidLite/QNN board demo", + "Input: six raw camera JPGs", + "Preprocess: BGR->RGB normalize resize CHW pad", + "Temporal: scene-start reset, then prev_bev recursion", + f"Score threshold: {score_thr}", + f"Max boxes: {max_boxes}", + "Red: W8A8 final detections", + ] + for raw in lines: + color = (0, 0, 0) + if raw.startswith("Final"): + color = (180, 0, 0) + elif raw.startswith("Red"): + color = (220, 20, 20) + for line in wrap_text(raw, width=42): + draw.text((x0, y), line, fill=color, font=body_font) + y += 27 + y += 4 + if result_dir is not None: + y += 14 + draw.text((x0, y), "Input result directory:", fill=(0, 0, 0), font=body_font) + y += 28 + for line in wrap_text(str(result_dir), width=48): + draw.text((x0, y), line, fill=(80, 80, 80), font=small_font) + y += 22 + return canvas + + +def save_camera_grid_visualization( + output_dir: str | Path, + frame_index: int, + frame_assets: dict[str, Any], + repo_root: str | Path, + boxes: np.ndarray, + scores: np.ndarray, + labels: np.ndarray, + score_thr: float = 0.25, + max_boxes: int = 80, + panel_width: int = 900, + draw_labels: bool = False, + metrics_panel: bool = True, + result_dir: str | Path | None = None, +) -> dict[str, Any]: + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + font = load_font(18) + label_font = load_font(15) + cameras = camera_records(frame_assets, repo_root) + lidar2img = load_lidar2img_by_camera(frame_assets, repo_root) + preds = select_predictions(boxes, scores, labels, score_thr, max_boxes) + + panels = [] + per_camera = {} + for camera_name in GRID_CAMERA_ORDER: + image = Image.open(cameras[camera_name]).convert("RGB").resize((800, 450)) + draw = ImageDraw.Draw(image) + count = 0 + for box, score, class_name in preds: + points = project_box(box, lidar2img[camera_name], image.size) + if points is None: + continue + draw_projected_box(draw, points, (255, 40, 40), width=3) + if draw_labels: + draw_label(draw, tuple(points[:, :2].min(axis=0)), f"W8A8 {class_name} {score:.2f}", (255, 180, 180), label_font) + count += 1 + header = Image.new("RGB", (image.width, 44), (20, 20, 20)) + hdraw = ImageDraw.Draw(header) + hdraw.text((10, 10), f"{camera_name} | W8A8 {count}", fill=(255, 255, 255), font=font) + panel = Image.new("RGB", (image.width, image.height + header.height), (0, 0, 0)) + panel.paste(header, (0, 0)) + panel.paste(image, (0, header.height)) + panel = panel.resize((panel_width, int(panel.height * panel_width / panel.width))) + panels.append(panel) + per_camera[camera_name] = {"w8a8": int(count)} + + cols, rows = 3, 2 + panel_w, panel_h = panels[0].size + title_h = 70 + sheet = Image.new("RGB", (cols * panel_w, rows * panel_h + title_h), (245, 245, 245)) + sheet_draw = ImageDraw.Draw(sheet) + branch = "scene_start" if bool(frame_assets.get("is_scene_start", False)) else "temporal" + token = str(frame_assets.get("sample_token", ""))[:16] + title = f"Sample4 frame{frame_index:03d} ({branch}) token={token} | red=W8A8 final | score_thr={score_thr}" + sheet_draw.text((16, 18), title, fill=(0, 0, 0), font=font) + for index, panel in enumerate(panels): + x = (index % cols) * panel_w + y = title_h + (index // cols) * panel_h + sheet.paste(panel, (x, y)) + + if metrics_panel: + sheet = add_sidebar(sheet, frame_index, branch, score_thr, max_boxes, result_dir) + + out_file = output_path / f"frame{frame_index:03d}_camera_grid.png" + sheet.save(out_file) + return { + "path": str(out_file), + "sha256": sha256_file(out_file), + "mode": "camera_grid_w8a8_only", + "score_thr": float(score_thr), + "max_boxes": int(max_boxes), + "candidate_count": int(len(preds)), + "per_camera_projected_counts": per_camera, + } + + +def save_camera_grid_gif(records: Sequence[dict[str, Any]], output_dir: str | Path, name: str = "sample4_camera_grid.gif", width: int = 1800, duration_ms: int = 700) -> dict[str, Any] | None: + paths = [Path(record["path"]) for record in records if record.get("path")] + if not paths: + return None + frames = [] + for path in paths: + image = Image.open(path).convert("RGB") + height = int(image.height * width / image.width) + frames.append(image.resize((width, height))) + out_file = Path(output_dir) / name + frames[0].save(out_file, save_all=True, append_images=frames[1:], duration=duration_ms, loop=0) + return {"path": str(out_file), "sha256": sha256_file(out_file), "frame_count": len(frames)} + + +def save_camera_grid_summary(output_dir: str | Path, records: Sequence[dict[str, Any]], gif_record: dict[str, Any] | None) -> dict[str, Any]: + output_path = Path(output_dir) + summary = { + "status": "PASS", + "mode": "sample4_camera_grid_w8a8_only", + "visualizations": list(records), + "gif": gif_record, + } + path = output_path / "sample4_camera_grid_summary.json" + path.write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + summary["path"] = str(path) + summary["sha256"] = sha256_file(path) + return summary diff --git a/targets/QCS8550/FP16/code/python/configs/demo_config.json b/targets/QCS8550/FP16/code/python/configs/demo_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a19b68455454bec5e4f74c423c526fb7983a0876 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/configs/demo_config.json @@ -0,0 +1,34 @@ +{ + "_comment": "BEVFormer board demo runtime config. Paths are relative to the bevformer_delivery_demo root.", + "name": "bevformer_delivery_demo", + "default_variant": "strict_w8a8_train64", + "model_type": "QNN248", + "models": { + "_comment": "Model paths are resolved automatically from models//FP16, or from the four CLI model path arguments. Supported platforms: QCS8550, IQ9, QCS8625, IQ8." + }, + "inputs": { + "_comment": "The bundled sample is four continuous frames from raw six-camera JPGs.", + "asset_manifest": "code/python/datasets/sample4/asset_manifest.json", + "frame_count": 4, + "scene_start_indices": [ + 0 + ], + "preprocess_workers": 6 + }, + "postprocess": { + "_comment": "NMSFreeCoder contract and saved detection output format.", + "nms_contract": "code/python/configs/nms_runtime_contract.json", + "output": "frameXXX_final_coordinates.npz" + }, + "visualization": { + "_comment": "Camera-grid visualization generated from W8A8 detections and bundled camera JPGs.", + "enabled_by_default": true, + "output": "frameXXX_camera_grid.png + sample4_camera_grid.gif", + "score_thr": 0.25, + "max_boxes": 80 + }, + "outputs": { + "_comment": "Default output root. If --output_dir is omitted, each run is saved under outputs/YYYY_MM_DD_HH_MM.", + "default_dir": "outputs" + } +} diff --git a/targets/QCS8550/FP16/code/python/configs/nms_runtime_contract.json b/targets/QCS8550/FP16/code/python/configs/nms_runtime_contract.json new file mode 100644 index 0000000000000000000000000000000000000000..41af80c6082057db8800da42ec9db64fa5318535 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/configs/nms_runtime_contract.json @@ -0,0 +1,11 @@ +{ + "type": "NMSFreeCoder", + "num_classes": 10, + "max_num": 300, + "num_query": 900, + "code_size": 10, + "pc_range": [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0], + "post_center_range": [-61.2, -61.2, -10.0, 61.2, 61.2, 10.0], + "score_threshold": null, + "selected_sigmoid_mode": "direct" +} diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/README.md b/targets/QCS8550/FP16/code/python/datasets/sample4/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ea20c2ad0a2cc040ba869d4aaed8a1662360e9f7 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/README.md @@ -0,0 +1,10 @@ +# BEVFormer Sample4 Dataset + +Self-contained four-frame continuous sample for the AidLite board demo. + +- Each frame contains six original nuScenes camera JPG files under `cameras/`. +- `python/utils.py` preprocesses those JPG files on the board before backbone inference. +- Small raw tensors are still included for BEVFormer metadata: `can_bus`, `shift`, `lidar2img`, and temporal rotation inputs. +- frame 000 uses `encoder_scene_start` semantics. +- frames 001-003 use `encoder_temporal` with the previous live `bev_embed`. +- Paths in `asset_manifest.json` are relative to `/home/aidlux` when copied to the board. diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/asset_manifest.json b/targets/QCS8550/FP16/code/python/datasets/sample4/asset_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..484cd94f1d21965b187e1e1794cf195dd87955a2 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/asset_manifest.json @@ -0,0 +1,535 @@ +{ + "audit_type": "BEVFORMER_DELIVERY_DEMO_SAMPLE4_MANIFEST", + "status": "PASS", + "mode": "self_contained_sample4_raw_camera_to_detection", + "total_frames": 4, + "scene_start_indices": [ + 0 + ], + "frame_range": [ + 0, + 3 + ], + "note": "Self-contained 4-frame continuous board demo sample. Image input starts from bundled raw camera JPEGs; Python preprocessing runs on board.", + "frames": { + "sample_000": { + "dataset_index": 0, + "sample_token": "3e8750f331d7499e9b5123e9eb70f2e2", + "scene_token": "fcbccedd61424f1b85dcbf8f897f9754", + "prev_token": "", + "next_token": "3950bd41f74548429c0f7700ff3d8269", + "is_scene_start": true, + "assets": { + "can_bus": { + "dtype": "float16", + "shape": [ + 1, + 18 + ], + "bytes": 36, + "path": "code/python/datasets/sample4/frames/sample_000/can_bus.raw", + "sha256": "974730a9cf60a4a0cae487ed4b296faf6621abec0bcd7276020ade042fd8dc3b" + }, + "lidar2img": { + "dtype": "float16", + "shape": [ + 1, + 6, + 4, + 4 + ], + "bytes": 192, + "path": "code/python/datasets/sample4/frames/sample_000/lidar2img.raw", + "sha256": "f1df04f672d707f2fbc8ea9799ee84d8795a1fe22f17ed89c08327e63f663501" + }, + "camera_images": { + "type": "raw_camera_jpeg_multiview", + "order": [ + "CAM_FRONT", + "CAM_FRONT_RIGHT", + "CAM_FRONT_LEFT", + "CAM_BACK", + "CAM_BACK_LEFT", + "CAM_BACK_RIGHT" + ], + "decode": "cv2.imread BGR", + "pipeline": "BGR -> RGB -> normalize(mean/std) -> resize(800,450) -> CHW -> pad to 480 rows", + "mean": [ + 123.675, + 116.28, + 103.53 + ], + "std": [ + 58.395, + 57.12, + 57.375 + ], + "to_rgb": true, + "resize": [ + 800, + 450 + ], + "output_shape_unpadded": [ + 6, + 3, + 450, + 800 + ], + "output_shape_padded": [ + 6, + 3, + 480, + 800 + ], + "reference_preprocessed_sha256": "87de4dfddc0cb554908b8c9a89d76125e8698bfbf5ece9d366de924a938d4b1d", + "images": [ + { + "name": "CAM_FRONT", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT.jpg", + "sha256": "0e2067a1618c4424050c45c393ada8c9dbf130fff8d6015004fce901dc07984d", + "bytes": 141976 + }, + { + "name": "CAM_FRONT_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_RIGHT.jpg", + "sha256": "15cf59c79b086d14da6fae5a7ee5b0f4d0fac0c0a22c0a85faaa5fb215521191", + "bytes": 147910 + }, + { + "name": "CAM_FRONT_LEFT", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_LEFT.jpg", + "sha256": "8c52f73a3ef06a129c27ef51ac65704428caebfaa60c52f14a5953bbb0124a18", + "bytes": 144177 + }, + { + "name": "CAM_BACK", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK.jpg", + "sha256": "047dd43f3f9fb7cc426b1d4ae369181242c5555b84baff5d7e1cbd0dc8b07259", + "bytes": 108796 + }, + { + "name": "CAM_BACK_LEFT", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_LEFT.jpg", + "sha256": "27b2826991733662e4f17c509ad5a66e85e0ca66a6413063d8db9645fa608c11", + "bytes": 110941 + }, + { + "name": "CAM_BACK_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_RIGHT.jpg", + "sha256": "a998caf0e327816b82d2a142d2581a8c1f9c4e67d50b4af3718a381337b7e415", + "bytes": 124547 + } + ] + } + } + }, + "sample_001": { + "dataset_index": 1, + "sample_token": "3950bd41f74548429c0f7700ff3d8269", + "scene_token": "fcbccedd61424f1b85dcbf8f897f9754", + "prev_token": "3e8750f331d7499e9b5123e9eb70f2e2", + "next_token": "c5f58c19249d4137ae063b0e9ecd8b8e", + "is_scene_start": false, + "assets": { + "can_bus": { + "dtype": "float16", + "shape": [ + 1, + 18 + ], + "bytes": 36, + "path": "code/python/datasets/sample4/frames/sample_001/can_bus.raw", + "sha256": "8ece2acd67155365b9394e1569b8242d69f3d9805cea19a5153cb51e79e55152" + }, + "shift": { + "dtype": "float16", + "shape": [ + 1, + 2 + ], + "bytes": 4, + "path": "code/python/datasets/sample4/frames/sample_001/shift.raw", + "sha256": "530492c17f13ce48ec402622565d7a4ba0987a57b988a3c4d824cc4987dfd5fc" + }, + "lidar2img": { + "dtype": "float16", + "shape": [ + 1, + 6, + 4, + 4 + ], + "bytes": 192, + "path": "code/python/datasets/sample4/frames/sample_001/lidar2img.raw", + "sha256": "63d7bcbf3a9c63e1184947d9e5112fe30bf4e8533ea29df44daeef3cb610a2ed" + }, + "rotation_can_bus": { + "dtype": "float32", + "shape": [ + 1, + 18 + ], + "bytes": 72, + "path": "code/python/datasets/sample4/frames/sample_001/rotation_can_bus.raw", + "sha256": "741d75aad896829e7246d446635bc21331242c7e28ef869f986d70e048d40962" + }, + "camera_images": { + "type": "raw_camera_jpeg_multiview", + "order": [ + "CAM_FRONT", + "CAM_FRONT_RIGHT", + "CAM_FRONT_LEFT", + "CAM_BACK", + "CAM_BACK_LEFT", + "CAM_BACK_RIGHT" + ], + "decode": "cv2.imread BGR", + "pipeline": "BGR -> RGB -> normalize(mean/std) -> resize(800,450) -> CHW -> pad to 480 rows", + "mean": [ + 123.675, + 116.28, + 103.53 + ], + "std": [ + 58.395, + 57.12, + 57.375 + ], + "to_rgb": true, + "resize": [ + 800, + 450 + ], + "output_shape_unpadded": [ + 6, + 3, + 450, + 800 + ], + "output_shape_padded": [ + 6, + 3, + 480, + 800 + ], + "reference_preprocessed_sha256": "ecc0a38ce961abecd5063c3b5c8cb543a63cb423c032ca06fbdc8efc73b66f4a", + "images": [ + { + "name": "CAM_FRONT", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT.jpg", + "sha256": "d5624fcecf12310c3d0760362e097c38cdcdce987e032aaf2883071cd14e2732", + "bytes": 146297 + }, + { + "name": "CAM_FRONT_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_RIGHT.jpg", + "sha256": "dd64d7463bc06bfd5d5378bdd494b40d68d0894307f3ed971eaaaf131022cf68", + "bytes": 143628 + }, + { + "name": "CAM_FRONT_LEFT", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_LEFT.jpg", + "sha256": "0fe15c0581db41e084df27acb2183b0ab9b0fd36da136ef1b4bcbda6f1a84c58", + "bytes": 146244 + }, + { + "name": "CAM_BACK", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK.jpg", + "sha256": "fdbdaa77e60ed5a6a762a4ea9a8eb1cb2ca9a3d94c610a55a2399a05c04b6ba6", + "bytes": 109601 + }, + { + "name": "CAM_BACK_LEFT", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_LEFT.jpg", + "sha256": "000f7943769e91ffa842a7dfd1bfc31e0c3f4b37325a98846e8261304028f2ad", + "bytes": 112425 + }, + { + "name": "CAM_BACK_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_RIGHT.jpg", + "sha256": "39d28a8a5a6f61370a7e7e1e965a6d8610b571195c50fd29da960e1fb2fbee80", + "bytes": 125712 + } + ] + } + } + }, + "sample_002": { + "dataset_index": 2, + "sample_token": "c5f58c19249d4137ae063b0e9ecd8b8e", + "scene_token": "fcbccedd61424f1b85dcbf8f897f9754", + "prev_token": "3950bd41f74548429c0f7700ff3d8269", + "next_token": "700c1a25559b4433be532de3475e58a9", + "is_scene_start": false, + "assets": { + "can_bus": { + "dtype": "float16", + "shape": [ + 1, + 18 + ], + "bytes": 36, + "path": "code/python/datasets/sample4/frames/sample_002/can_bus.raw", + "sha256": "e8badd418a0eb1905986c2a2a5e87fa6e16532dc4899cfff93f47dbe51bed9be" + }, + "shift": { + "dtype": "float16", + "shape": [ + 1, + 2 + ], + "bytes": 4, + "path": "code/python/datasets/sample4/frames/sample_002/shift.raw", + "sha256": "9cd32a1a7fbb67a9841ebee6acf497db1fa6142ec9e3ec5c9bb1596dd8be81e8" + }, + "lidar2img": { + "dtype": "float16", + "shape": [ + 1, + 6, + 4, + 4 + ], + "bytes": 192, + "path": "code/python/datasets/sample4/frames/sample_002/lidar2img.raw", + "sha256": "7d524fefbeec4c1014ab2773b722aaa13774cb4787fb950e27b1b4de7cc3d403" + }, + "rotation_can_bus": { + "dtype": "float32", + "shape": [ + 1, + 18 + ], + "bytes": 72, + "path": "code/python/datasets/sample4/frames/sample_002/rotation_can_bus.raw", + "sha256": "a08e149204cd22c460e8b5cc62726de1b29d22fb05b6c33b450e1f94a162d3c1" + }, + "camera_images": { + "type": "raw_camera_jpeg_multiview", + "order": [ + "CAM_FRONT", + "CAM_FRONT_RIGHT", + "CAM_FRONT_LEFT", + "CAM_BACK", + "CAM_BACK_LEFT", + "CAM_BACK_RIGHT" + ], + "decode": "cv2.imread BGR", + "pipeline": "BGR -> RGB -> normalize(mean/std) -> resize(800,450) -> CHW -> pad to 480 rows", + "mean": [ + 123.675, + 116.28, + 103.53 + ], + "std": [ + 58.395, + 57.12, + 57.375 + ], + "to_rgb": true, + "resize": [ + 800, + 450 + ], + "output_shape_unpadded": [ + 6, + 3, + 450, + 800 + ], + "output_shape_padded": [ + 6, + 3, + 480, + 800 + ], + "reference_preprocessed_sha256": "97fc26501e167d58279ce6229847cd6753b13b2b1795effba7dcbf282f99fc2e", + "images": [ + { + "name": "CAM_FRONT", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT.jpg", + "sha256": "0bb82b89f9e6ce47d9e1bcd747b01fe3fd342ccc67eb7a9c693a3fc3fc73a6bc", + "bytes": 145856 + }, + { + "name": "CAM_FRONT_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_RIGHT.jpg", + "sha256": "70856b9d713c279b4b3d5911be30944f339760cc81da5a8f2c4d90a20276c1d3", + "bytes": 132652 + }, + { + "name": "CAM_FRONT_LEFT", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_LEFT.jpg", + "sha256": "4782aaa9180ef1fe153e702031a3249ada09bdb056ad13ff69abaa2f9e59e35a", + "bytes": 145095 + }, + { + "name": "CAM_BACK", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK.jpg", + "sha256": "a9a05b52aea16ccf730447170eb836697c26928a112a74249caf423d91d28306", + "bytes": 111521 + }, + { + "name": "CAM_BACK_LEFT", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_LEFT.jpg", + "sha256": "a28438e9ff68ccab2ce3a8da4151591a2b15780aecdc21185a911318ed923f72", + "bytes": 112445 + }, + { + "name": "CAM_BACK_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_RIGHT.jpg", + "sha256": "1578ba72afd2cc91abb1295c3404a432e41ffc7385b4031a43b586acfb693872", + "bytes": 132515 + } + ] + } + } + }, + "sample_003": { + "dataset_index": 3, + "sample_token": "700c1a25559b4433be532de3475e58a9", + "scene_token": "fcbccedd61424f1b85dcbf8f897f9754", + "prev_token": "c5f58c19249d4137ae063b0e9ecd8b8e", + "next_token": "747aa46b9a4641fe90db05d97db2acea", + "is_scene_start": false, + "assets": { + "can_bus": { + "dtype": "float16", + "shape": [ + 1, + 18 + ], + "bytes": 36, + "path": "code/python/datasets/sample4/frames/sample_003/can_bus.raw", + "sha256": "dbc4945c65d5804131f75f04578e3fbc288330ff5fa977f224eda07eb96e463a" + }, + "shift": { + "dtype": "float16", + "shape": [ + 1, + 2 + ], + "bytes": 4, + "path": "code/python/datasets/sample4/frames/sample_003/shift.raw", + "sha256": "8cc4c98a9615a420d40104ae51d4f1f50b719a9cfec4be8d1b65b4b704517def" + }, + "lidar2img": { + "dtype": "float16", + "shape": [ + 1, + 6, + 4, + 4 + ], + "bytes": 192, + "path": "code/python/datasets/sample4/frames/sample_003/lidar2img.raw", + "sha256": "64cd7874438918c64f54414d1b873b0b0fcf05b2b601f2014c4fe5a8316e7a2c" + }, + "rotation_can_bus": { + "dtype": "float32", + "shape": [ + 1, + 18 + ], + "bytes": 72, + "path": "code/python/datasets/sample4/frames/sample_003/rotation_can_bus.raw", + "sha256": "3c88129317e663a14f4ba55a3c0c3cc25e9ec2cd4c497276567e4ecfed8a9ab6" + }, + "camera_images": { + "type": "raw_camera_jpeg_multiview", + "order": [ + "CAM_FRONT", + "CAM_FRONT_RIGHT", + "CAM_FRONT_LEFT", + "CAM_BACK", + "CAM_BACK_LEFT", + "CAM_BACK_RIGHT" + ], + "decode": "cv2.imread BGR", + "pipeline": "BGR -> RGB -> normalize(mean/std) -> resize(800,450) -> CHW -> pad to 480 rows", + "mean": [ + 123.675, + 116.28, + 103.53 + ], + "std": [ + 58.395, + 57.12, + 57.375 + ], + "to_rgb": true, + "resize": [ + 800, + 450 + ], + "output_shape_unpadded": [ + 6, + 3, + 450, + 800 + ], + "output_shape_padded": [ + 6, + 3, + 480, + 800 + ], + "reference_preprocessed_sha256": "da157bc039b66254cb57f827adcdfde783d324a078a15af9740a3701697eba15", + "images": [ + { + "name": "CAM_FRONT", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT.jpg", + "sha256": "54384bc8c4896f349bdc643b573f398fbb03ee842ac2af2af2c602a62f48f79d", + "bytes": 147138 + }, + { + "name": "CAM_FRONT_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_RIGHT.jpg", + "sha256": "4c5209d28a7e3a5f29bd7c02c29bd0f53c669afc43b17af2db098371a9711dc1", + "bytes": 145853 + }, + { + "name": "CAM_FRONT_LEFT", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_LEFT.jpg", + "sha256": "1f20956c425bdc84da1bf2610b7214d48930652782394aa0f1f3bef94ce0be35", + "bytes": 146337 + }, + { + "name": "CAM_BACK", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK.jpg", + "sha256": "34e84e10e04ab229864bd0ddd3067442c0b58e249eeb48088ab417cb259b4985", + "bytes": 112516 + }, + { + "name": "CAM_BACK_LEFT", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_LEFT.jpg", + "sha256": "cdf1f064c61add58e17d7011713aebbe40be9d7e8290c8816f585fca3841e033", + "bytes": 113884 + }, + { + "name": "CAM_BACK_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_RIGHT.jpg", + "sha256": "eae354ba8407662caa6c50934b6ee306a4e5aaaeec0b0f2e40c2f6b481485d3f", + "bytes": 135629 + } + ] + } + } + } + }, + "contract": { + "image_pipeline": "cv2.imread BGR -> RGB -> normalize(mean=[123.675,116.28,103.53], std=[58.395,57.12,57.375]) -> cv2.resize(800,450) -> CHW -> zero-pad to (6,3,480,800)", + "camera_order": [ + "CAM_FRONT", + "CAM_FRONT_RIGHT", + "CAM_FRONT_LEFT", + "CAM_BACK", + "CAM_BACK_LEFT", + "CAM_BACK_RIGHT" + ], + "model_path": "W8A8 backbone + scene-start/temporal encoder + decoder AidLite QNN contexts", + "temporal_state": "scene-start uses reset semantics; temporal frames use previous live bev_embed as prev_bev", + "scene_start_inputs": "camera_images + can_bus + lidar2img; no prev_bev/shift input is required for scene-start encoder", + "temporal_inputs": "camera_images + can_bus + lidar2img + shift + rotated previous live bev_embed" + } +} diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bbe7c64a07c851a2a41c38e6cd377a6db50dde30 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:047dd43f3f9fb7cc426b1d4ae369181242c5555b84baff5d7e1cbd0dc8b07259 +size 108796 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_LEFT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_LEFT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5a0335a8869e040adea7d32e9add84b2d9981627 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_LEFT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27b2826991733662e4f17c509ad5a66e85e0ca66a6413063d8db9645fa608c11 +size 110941 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_RIGHT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_RIGHT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..afaafc6d7457adb1b59619e50232277a39f9d253 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_RIGHT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a998caf0e327816b82d2a142d2581a8c1f9c4e67d50b4af3718a381337b7e415 +size 124547 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..36ad1e76aec369b9d84fe5ed054e9f74648fcb62 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e2067a1618c4424050c45c393ada8c9dbf130fff8d6015004fce901dc07984d +size 141976 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_LEFT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_LEFT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..603c729bdc57e650b1a5505c313d7dab4b2684c4 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_LEFT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c52f73a3ef06a129c27ef51ac65704428caebfaa60c52f14a5953bbb0124a18 +size 144177 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_RIGHT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_RIGHT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..abcaea332dcb152e71346be1026033e942e2b93a --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_RIGHT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15cf59c79b086d14da6fae5a7ee5b0f4d0fac0c0a22c0a85faaa5fb215521191 +size 147910 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/can_bus.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/can_bus.raw new file mode 100644 index 0000000000000000000000000000000000000000..c44485c50b522b89428688462584f99e732b1cb9 Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/can_bus.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/lidar2img.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/lidar2img.raw new file mode 100644 index 0000000000000000000000000000000000000000..a001cde0826dbbbbfe3610ec2dceb618f314437e Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_000/lidar2img.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c753c7c1ced6671589f5d78ef80b036ee74579c7 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdbdaa77e60ed5a6a762a4ea9a8eb1cb2ca9a3d94c610a55a2399a05c04b6ba6 +size 109601 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_LEFT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_LEFT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c11bcead8a04f708c3b8b9ace60c527d3d8fbc6a --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_LEFT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:000f7943769e91ffa842a7dfd1bfc31e0c3f4b37325a98846e8261304028f2ad +size 112425 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_RIGHT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_RIGHT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e4e0e06fcc31b2d935d104907f24a5a926a76c74 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_RIGHT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d28a8a5a6f61370a7e7e1e965a6d8610b571195c50fd29da960e1fb2fbee80 +size 125712 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b86497ee36d90a831642dd8a6426ff07ad9c4321 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5624fcecf12310c3d0760362e097c38cdcdce987e032aaf2883071cd14e2732 +size 146297 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_LEFT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_LEFT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5558efea4739e44b8c5f97c6a370cf1e2e3d9463 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_LEFT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fe15c0581db41e084df27acb2183b0ab9b0fd36da136ef1b4bcbda6f1a84c58 +size 146244 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_RIGHT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_RIGHT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..13a14b953437454305df2d6e885af3c96fcec1ee --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_RIGHT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd64d7463bc06bfd5d5378bdd494b40d68d0894307f3ed971eaaaf131022cf68 +size 143628 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/can_bus.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/can_bus.raw new file mode 100644 index 0000000000000000000000000000000000000000..997ffb0bb9a1951b80e21b2059ef0c204c579727 Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/can_bus.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/lidar2img.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/lidar2img.raw new file mode 100644 index 0000000000000000000000000000000000000000..2aac1d738f37c59353a36987cf908f408733aa24 Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/lidar2img.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/rotation_can_bus.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/rotation_can_bus.raw new file mode 100644 index 0000000000000000000000000000000000000000..23118c376c799fd8ff34e1e115d4406d7f9ef627 Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/rotation_can_bus.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/shift.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/shift.raw new file mode 100644 index 0000000000000000000000000000000000000000..16636bb3f7bd82d4de14134c8f8522da5bad2a9c --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_001/shift.raw @@ -0,0 +1 @@ +T) \ No newline at end of file diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK.jpg new file mode 100644 index 0000000000000000000000000000000000000000..119d37589070644b37c98d7231a26ca5c5b4ae67 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9a05b52aea16ccf730447170eb836697c26928a112a74249caf423d91d28306 +size 111521 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_LEFT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_LEFT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8f009ffcf2e0276633c46bae846cbeb1b8e6a21c --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_LEFT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a28438e9ff68ccab2ce3a8da4151591a2b15780aecdc21185a911318ed923f72 +size 112445 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_RIGHT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_RIGHT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dc0d588f8bd4f35d9ac401b998780f9805130390 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_RIGHT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1578ba72afd2cc91abb1295c3404a432e41ffc7385b4031a43b586acfb693872 +size 132515 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e5f5ba82f8111fbc2d62d032eca64b3d19a5478d --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bb82b89f9e6ce47d9e1bcd747b01fe3fd342ccc67eb7a9c693a3fc3fc73a6bc +size 145856 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_LEFT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_LEFT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4a9f6d7951d9939687139cd59ab1dd668be7f77a --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_LEFT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4782aaa9180ef1fe153e702031a3249ada09bdb056ad13ff69abaa2f9e59e35a +size 145095 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_RIGHT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_RIGHT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..663c25012d40f2e2694617c2e07aaac8cb3015fd --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_RIGHT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70856b9d713c279b4b3d5911be30944f339760cc81da5a8f2c4d90a20276c1d3 +size 132652 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/can_bus.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/can_bus.raw new file mode 100644 index 0000000000000000000000000000000000000000..aad037b2fff71f0c31492bdb13df6fc65e03573d Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/can_bus.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/lidar2img.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/lidar2img.raw new file mode 100644 index 0000000000000000000000000000000000000000..85b4d06504f799d7e982f9afbc1a7165758757ef Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/lidar2img.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/rotation_can_bus.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/rotation_can_bus.raw new file mode 100644 index 0000000000000000000000000000000000000000..f750ba0c2ec9b596aeb185b24d77601f7eea1b87 Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/rotation_can_bus.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/shift.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/shift.raw new file mode 100644 index 0000000000000000000000000000000000000000..4a7ae9a47e74e18f181177a5bf8dc35bff92dc36 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_002/shift.raw @@ -0,0 +1 @@ +>I) \ No newline at end of file diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK.jpg new file mode 100644 index 0000000000000000000000000000000000000000..47e053034a8cd13e2d0ffc43c2941e94e6845508 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34e84e10e04ab229864bd0ddd3067442c0b58e249eeb48088ab417cb259b4985 +size 112516 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_LEFT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_LEFT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9553893b753fefab54b445f13e56201861354b39 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_LEFT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdf1f064c61add58e17d7011713aebbe40be9d7e8290c8816f585fca3841e033 +size 113884 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_RIGHT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_RIGHT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b0bc6048c52f5c1e8d6df13f72db5bf2a7ce58b5 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_RIGHT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eae354ba8407662caa6c50934b6ee306a4e5aaaeec0b0f2e40c2f6b481485d3f +size 135629 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5169efba71bc8857941c64fc90664971b988dba4 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54384bc8c4896f349bdc643b573f398fbb03ee842ac2af2af2c602a62f48f79d +size 147138 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_LEFT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_LEFT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ee667054d9ea2baa07fd5b80b75916e54484a6f7 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_LEFT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f20956c425bdc84da1bf2610b7214d48930652782394aa0f1f3bef94ce0be35 +size 146337 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_RIGHT.jpg b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_RIGHT.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a5ae505809bb1fe09e493c83da040a00521b4670 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_RIGHT.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c5209d28a7e3a5f29bd7c02c29bd0f53c669afc43b17af2db098371a9711dc1 +size 145853 diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/can_bus.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/can_bus.raw new file mode 100644 index 0000000000000000000000000000000000000000..58f944d60504a7cb8f028f6e38c385f16a50dcf2 Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/can_bus.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/lidar2img.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/lidar2img.raw new file mode 100644 index 0000000000000000000000000000000000000000..a7514f9eade0e96a7407a43a06d3077ba4a80bac Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/lidar2img.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/rotation_can_bus.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/rotation_can_bus.raw new file mode 100644 index 0000000000000000000000000000000000000000..7970eabb4f2b0af7ead3cb195d975cfaca785770 Binary files /dev/null and b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/rotation_can_bus.raw differ diff --git a/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/shift.raw b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/shift.raw new file mode 100644 index 0000000000000000000000000000000000000000..075891c564ed0fd62a5df3c0fcc3d634e05218d1 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/datasets/sample4/frames/sample_003/shift.raw @@ -0,0 +1 @@ +8) \ No newline at end of file diff --git a/targets/QCS8550/FP16/code/python/docs/BOARD_ENVIRONMENT.md b/targets/QCS8550/FP16/code/python/docs/BOARD_ENVIRONMENT.md new file mode 100644 index 0000000000000000000000000000000000000000..1139adb82e2555f04bdb5191642b67429491170b --- /dev/null +++ b/targets/QCS8550/FP16/code/python/docs/BOARD_ENVIRONMENT.md @@ -0,0 +1,117 @@ +# Board Environment Dependencies + +This demo is intended to run on the AidLux board environment where the AidLite QNN240 runtime is available. A normal development container can run `--dry_run`, but cannot execute the QNN contexts unless `import aidlite` succeeds. + +## Verified Board Environment + +The current board used for validation reports: + +```text +Python 3.10.12 +aidlite import: OK +numpy 1.26.4 +opencv-python / cv2 4.13.0 +Pillow / PIL 10.4.0 +``` + +## Required Runtime Components + +- AidLux / AidLite Python runtime with `aidlite` module. +- AidLite QNN240 plugin/runtime for `FrameworkType.TYPE_QNN240`. +- Qualcomm HTP/DSP runtime and valid board license. +- Board-side QNN context execution support for the selected model platform. + +The Python code checks the AidLite enum contract at startup: + +```text +FrameworkType.TYPE_QNN240 = 109 +ImplementType.TYPE_LOCAL = 3 +AccelerateType.TYPE_DSP = 3 +``` + +## Required Python Packages + +The demo imports these Python packages at runtime: + +```text +numpy +cv2 +PIL +aidlite +``` + +Package usage: + +- `aidlite`: loads and invokes the four QNN240 context `.aidem` files. +- `numpy`: tensor loading, dtype conversion, BEVFormer postprocess, NPZ output. +- `cv2`: board-side six-camera JPG preprocessing. +- `PIL`: camera-grid PNG/GIF visualization. + +## Model/Data Files Required By Default Run + +Default command: + +```bash +python3 code/python/run_test.py --invoke_nums 4 --output_dir outputs/final_sample4 +``` + +The default command auto-discovers the first complete model directory in this order: + +```text +models/QCS8550/FP16 +models/IQ9/FP16 +models/QCS8625/FP16 +models/IQ8/FP16 +``` + +Model SHA verification is always enabled. The platform is inferred from model paths: + +```text +QCS8550 -> QCS8550 model SHA table +IQ9 -> QCS8550 model SHA table +QCS8625 -> QCS8625 model SHA table +IQ8 -> QCS8625 model SHA table +``` + +Each platform directory must contain: + +```text +backbone_context.bin.aidem +scene_start_encoder_context.bin.aidem +temporal_encoder_context.bin.aidem +decoder_context.bin.aidem +``` + +Required config files: + +```text +code/python/configs/demo_config.json +code/python/configs/nms_runtime_contract.json +``` + +Required sample data: + +```text +code/python/datasets/sample4/asset_manifest.json +datasets/sample4/frames/sample_000..sample_003/ +``` + +Each sample frame contains six raw camera JPGs plus the small auxiliary tensors required by the deployed contexts. + +## Quick Environment Check + +Run on the board: + +```bash +cd /home/aidlux/bevformer_delivery_demo +python3 -c "import aidlite, numpy, cv2; from PIL import Image; print('board env ok')" +python3 code/python/run_test.py --dry_run --check_raw_assets +``` + +The dry-run checks paths, model SHA, sample assets, and scene-start/temporal routing. It does not invoke AidLite/DSP. + +## Notes + +- Do not expect real inference to work in a host/container environment without AidLite. +- `configs/qnn_htp_configs/` is not required by runtime; it was only an optional encryption/audit supplement and is not included in the runtime demo package. +- Full inference timing and visualization are printed by the default command without adding extra timing flags. diff --git a/targets/QCS8550/FP16/code/python/portable_numpy_nmsfreecoder.py b/targets/QCS8550/FP16/code/python/portable_numpy_nmsfreecoder.py new file mode 100644 index 0000000000000000000000000000000000000000..ed3c107ada285ac23366a4ec821b5838760048c1 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/portable_numpy_nmsfreecoder.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Portable NumPy implementation of the BEVFormer NMSFreeCoder decode path.""" + +from __future__ import annotations + +import numpy as np + + +def _sigmoid_direct(value): + value = np.asarray(value, dtype=np.float32) + one = np.float32(1.0) + return one / (one + np.exp(-value)) + + +def _sigmoid_stable(value): + value = np.asarray(value, dtype=np.float32) + output = np.empty_like(value, dtype=np.float32) + + positive = value >= np.float32(0.0) + negative = ~positive + + output[positive] = np.float32(1.0) / ( + np.float32(1.0) + np.exp(-value[positive]) + ) + + exp_value = np.exp(value[negative]) + output[negative] = exp_value / ( + np.float32(1.0) + exp_value + ) + + return output + + +def _denormalize_bbox(normalized_bboxes): + boxes = np.asarray(normalized_bboxes, dtype=np.float32) + + if boxes.ndim != 2: + raise ValueError( + "bbox tensor must be 2D, got {}".format(boxes.shape) + ) + + if boxes.shape[1] < 8: + raise ValueError( + "bbox code size must be >=8, got {}".format(boxes.shape[1]) + ) + + cx = boxes[:, 0:1] + cy = boxes[:, 1:2] + width = np.exp(boxes[:, 2:3]) + length = np.exp(boxes[:, 3:4]) + cz = boxes[:, 4:5] + height = np.exp(boxes[:, 5:6]) + + rotation = np.arctan2( + boxes[:, 6:7], + boxes[:, 7:8], + ) + + if boxes.shape[1] > 8: + if boxes.shape[1] < 10: + raise ValueError( + "velocity bbox code requires 10 values, got {}".format( + boxes.shape[1] + ) + ) + + velocity_x = boxes[:, 8:9] + velocity_y = boxes[:, 9:10] + + return np.ascontiguousarray( + np.concatenate( + [ + cx, + cy, + cz, + width, + length, + height, + rotation, + velocity_x, + velocity_y, + ], + axis=-1, + ), + dtype=np.float32, + ) + + return np.ascontiguousarray( + np.concatenate( + [ + cx, + cy, + cz, + width, + length, + height, + rotation, + ], + axis=-1, + ), + dtype=np.float32, + ) + + +def decode_numpy_nmsfreecoder( + cls_scores, + bbox_preds, + contract, + sigmoid_mode=None, + sort_kind=None, + precomputed_probabilities=None, +): + num_classes = int(contract["num_classes"]) + max_num = int(contract["max_num"]) + num_query = int(contract["num_query"]) + code_size = int(contract["code_size"]) + + bbox = np.asarray(bbox_preds, dtype=np.float32).reshape( + num_query, + code_size, + ) + + if not np.isfinite(bbox).all(): + raise ValueError("bbox predictions contain non-finite values") + + if precomputed_probabilities is not None: + probabilities = np.asarray( + precomputed_probabilities, + dtype=np.float32, + ).reshape(num_query, num_classes) + else: + cls = np.asarray(cls_scores, dtype=np.float32).reshape( + num_query, + num_classes, + ) + + if not np.isfinite(cls).all(): + raise ValueError("classification logits contain non-finite values") + + selected_sigmoid = ( + sigmoid_mode + or contract.get("selected_sigmoid_mode") + or "direct" + ) + + if selected_sigmoid == "direct": + probabilities = _sigmoid_direct(cls) + elif selected_sigmoid == "stable": + probabilities = _sigmoid_stable(cls) + else: + raise ValueError( + "unsupported sigmoid mode: {}".format(selected_sigmoid) + ) + + flattened = np.ascontiguousarray( + probabilities.reshape(-1), + dtype=np.float32, + ) + + keep_count = min(max_num, flattened.size) + + # Use argpartition (quickselect, same family as PyTorch CPU topk) to + # isolate the top-k candidates, then deterministically sort by + # (score descending, original index ascending) so ties are portable. + partition_index = keep_count - 1 + candidate_indices = np.argpartition( + -flattened, + partition_index, + kind="introselect", + )[:keep_count] + candidate_scores = flattened[candidate_indices] + + # Sort candidates by (-score, index) for deterministic tie-breaking. + sort_order = np.lexsort( + (candidate_indices, -candidate_scores) + ) + order = candidate_indices[sort_order] + + scores = np.ascontiguousarray( + flattened[order], + dtype=np.float32, + ) + labels = np.ascontiguousarray( + order % num_classes, + dtype=np.int64, + ) + bbox_indices = order // num_classes + + selected_bbox = np.ascontiguousarray( + bbox[bbox_indices], + dtype=np.float32, + ) + + decoded_boxes = _denormalize_bbox(selected_bbox) + + score_threshold = contract.get("score_threshold") + threshold_mask = np.ones( + scores.shape, + dtype=bool, + ) + + if score_threshold is not None: + threshold = float(score_threshold) + threshold_mask = scores > np.float32(threshold) + + temporary_threshold = threshold + + while int(np.count_nonzero(threshold_mask)) == 0: + temporary_threshold *= 0.9 + + if temporary_threshold < 0.01: + threshold_mask = scores > np.float32(-1.0) + break + + threshold_mask = scores >= np.float32( + temporary_threshold + ) + + post_center_range = contract.get("post_center_range") + + if post_center_range is None: + raise ValueError( + "post_center_range is required by this BEVFormer contract" + ) + + post_center = np.asarray( + post_center_range, + dtype=np.float32, + ).reshape(-1) + + if post_center.size != 6: + raise ValueError( + "post_center_range must contain 6 values" + ) + + spatial_mask = np.all( + decoded_boxes[:, :3] >= post_center[:3], + axis=1, + ) + spatial_mask &= np.all( + decoded_boxes[:, :3] <= post_center[3:], + axis=1, + ) + + # Match the official implementation's truth-value condition. + if score_threshold: + spatial_mask &= threshold_mask + + final_boxes = np.ascontiguousarray( + decoded_boxes[spatial_mask], + dtype=np.float32, + ) + final_scores = np.ascontiguousarray( + scores[spatial_mask], + dtype=np.float32, + ) + final_labels = np.ascontiguousarray( + labels[spatial_mask], + dtype=np.int64, + ) + + return final_boxes, final_scores, final_labels diff --git a/targets/QCS8550/FP16/code/python/run_demo.py b/targets/QCS8550/FP16/code/python/run_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..e60a3b12804a1db0d82548435642d16b0100b105 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/run_demo.py @@ -0,0 +1,592 @@ +from __future__ import annotations + +"""BEVFormer board demo command-line entry. + +This file is the orchestration layer of the demo. It does not implement the +model math itself. Its responsibilities are: + +1. Parse command-line arguments. +2. Resolve model/config/data/output paths. +3. Optionally run a dry-run check without loading AidLite. +4. Load the four QNN248 context files through :class:`BevFormerModel`. +5. Run the four-frame sample4 manifest and write a summary JSON. + +The real model execution lives in ``bevformer.py``. Image preprocessing and +small utility functions live in ``utils.py``. +""" + +import argparse +from datetime import datetime +import json +import sys +import time +from pathlib import Path + +from bevformer import MODEL_NAMES, MODEL_SHA256, BevFormerModel +from utils import EXPECTED_TENSORS, sha256_file + + +# ``code/python/run_demo.py`` is placed under the packaged Python demo. +# PACKAGE_DIR: .../BevFormer-Tiny-Resnet50/code/python +# CODE_ROOT : .../BevFormer-Tiny-Resnet50/code +# REPO_ROOT : .../BevFormer-Tiny-Resnet50 +PACKAGE_DIR = Path(__file__).resolve().parent +CODE_ROOT = PACKAGE_DIR.parent +REPO_ROOT = CODE_ROOT.parent +DEMO_ROOT = REPO_ROOT + +# Default QNN248 AidLite context files are discovered from models//FP16. +MODEL_FILE_NAMES = { + "backbone": "backbone_context.bin.aidem", + "encoder_temporal": "temporal_encoder_context.bin.aidem", + "encoder_scene_start": "scene_start_encoder_context.bin.aidem", + "decoder": "decoder_context.bin.aidem", +} +PREFERRED_MODEL_PLATFORMS = ("QCS8550", "IQ9", "QCS8625", "IQ8") + +# Default config, sample manifest, postprocess contract, and output directory. +DEFAULT_CONFIG = PACKAGE_DIR / "configs" / "demo_config.json" +DEFAULT_MANIFEST = PACKAGE_DIR / "datasets" / "sample4" / "asset_manifest.json" +DEFAULT_NMS_CONTRACT = PACKAGE_DIR / "configs" / "nms_runtime_contract.json" +DEFAULT_OUTPUT = REPO_ROOT / "outputs" + +# The demo always preprocesses the six camera JPGs in parallel by default. +DEFAULT_PREPROCESS_WORKERS = 6 + + +def parse_args() -> argparse.Namespace: + """Parse the command-line interface used by ``python/run_test.py``. + + ``run_test.py`` is only a thin wrapper. All actual CLI options are defined + here so the board command remains similar to a YOLOv5-style demo command. + """ + + parser = argparse.ArgumentParser(description="Run BEVFormer strict board demo with AidLite QNN248.") + + # Path arguments. If model paths are omitted, a complete models//FP16 + # directory is selected automatically. + parser.add_argument("--config", default=str(DEFAULT_CONFIG)) + parser.add_argument("--backbone_model") + parser.add_argument("--encoder_model") + parser.add_argument("--scene_start_encoder_model") + parser.add_argument("--decoder_model") + parser.add_argument("--asset_manifest") + parser.add_argument("--nms_contract") + parser.add_argument("--output_dir") + + # Frame range. ``--invoke_nums`` is kept as the YOLOv5-style alias for + # "how many samples to run". + parser.add_argument("--frame_start", type=int, default=0) + parser.add_argument("--frame_count", type=int, default=4) + parser.add_argument( + "--invoke_nums", + type=int, + default=None, + help="YOLOv5-style alias for how many consecutive frames to run.", + ) + + # Output and visualization switches. + parser.add_argument("--save_all_raw", action="store_true") + parser.add_argument("--no_visualize", action="store_true", help="Disable camera-grid visualization image output.") + parser.add_argument("--vis_score_thr", type=float, default=0.0) + parser.add_argument("--vis_max_boxes", type=int, default=80) + + # SHA checking for JPG files is optional because it adds extra file I/O. + # Model context SHA checking is always performed. + parser.add_argument( + "--check_image_sha", + action="store_true", + help="Verify every camera JPG SHA during real inference. Slower; useful for audit runs.", + ) + + # Only QNN248 is supported by this delivery package. + parser.add_argument("--model_type", default="QNN248") + + # Dry-run mode is for host/package checks. It avoids AidLite loading and + # therefore can run on a normal development machine. + parser.add_argument( + "--dry_run", + action="store_true", + help="Inspect config, model SHA, manifest, and scene/temporal routing without loading AidLite.", + ) + parser.add_argument( + "--check_raw_assets", + action="store_true", + help="In dry-run mode, also check that every raw asset path referenced by the selected frames exists.", + ) + return parser.parse_args() + + +def require_file(name: str, path: str) -> str: + """Return an absolute file path after existence and non-empty checks.""" + + value = Path(path).expanduser().resolve() + if not value.is_file() or value.stat().st_size == 0: + raise FileNotFoundError(f"{name} missing or empty: {value}") + return str(value) + + +def load_config(path: str) -> dict: + """Load the main JSON config used to find default model/data paths.""" + + value = Path(path).expanduser().resolve() + if not value.is_file(): + raise FileNotFoundError(value) + return json.loads(value.read_text(encoding="utf-8")) + + +def demo_path(config: dict, key_path: tuple[str, ...], fallback: Path | None = None) -> str | None: + """Resolve one optional path from ``demo_config.json``. + + Relative paths in the config are interpreted relative to ``DEMO_ROOT``. + If the key is missing and no fallback is supplied, the function returns None. + """ + + current = config + for key in key_path: + if not isinstance(current, dict) or key not in current: + return str(fallback) if fallback is not None else None + current = current[key] + + path = Path(str(current)) + if not path.is_absolute(): + path = DEMO_ROOT / path + return str(path) + + +def model_paths_for_platform(platform: str) -> dict[str, str]: + model_root = REPO_ROOT / "models" / platform / "FP16" + return {name: str(model_root / filename) for name, filename in MODEL_FILE_NAMES.items()} + + +def is_complete_model_platform(platform: str) -> bool: + return all(Path(path).is_file() for path in model_paths_for_platform(platform).values()) + + +def discover_default_model_paths() -> tuple[str, dict[str, str]]: + available = [platform for platform in PREFERRED_MODEL_PLATFORMS if is_complete_model_platform(platform)] + if not available: + expected = ", ".join(f"models/{platform}/FP16" for platform in PREFERRED_MODEL_PLATFORMS) + raise FileNotFoundError(f"No complete model directory found. Expected one of: {expected}") + platform = available[0] + return platform, model_paths_for_platform(platform) + + +def resolve_model_paths(args: argparse.Namespace, config: dict) -> tuple[str, dict[str, str]]: + cli_paths = { + "backbone": args.backbone_model, + "encoder_temporal": args.encoder_model, + "encoder_scene_start": args.scene_start_encoder_model, + "decoder": args.decoder_model, + } + if any(cli_paths.values()): + missing = [name for name, value in cli_paths.items() if not value] + if missing: + raise ValueError(f"When overriding model paths, provide all four model files. Missing: {missing}") + platform = BevFormerModel.resolve_model_platform(cli_paths) + return platform, {name: str(path) for name, path in cli_paths.items()} + + config_paths = { + "backbone": demo_path(config, ("models", "backbone")), + "encoder_temporal": demo_path(config, ("models", "encoder_temporal")), + "encoder_scene_start": demo_path(config, ("models", "encoder_scene_start")), + "decoder": demo_path(config, ("models", "decoder")), + } + if any(config_paths.values()): + missing = [name for name, value in config_paths.items() if not value] + if missing: + raise ValueError(f"demo_config.json model paths are incomplete. Missing: {missing}") + platform = BevFormerModel.resolve_model_platform(config_paths) + return platform, {name: str(path) for name, path in config_paths.items()} + + return discover_default_model_paths() + + +def _resolve_repo_path(path: str | Path) -> Path: + """Resolve manifest asset paths. + + The sample manifest stores board-style relative paths such as + ``bevformer_delivery_demo/datasets/...``. These are resolved under + ``REPO_ROOT`` so the same manifest works when the board package is located + at ``/home/aidlux/bevformer_delivery_demo``. + """ + + value = Path(path) + if value.is_absolute(): + return value + return REPO_ROOT / value + + +def _dry_run( + *, + backbone_model: str, + encoder_model: str, + scene_start_encoder_model: str, + decoder_model: str, + asset_manifest: str, + nms_contract: str, + output_dir: Path, + frame_start: int, + frame_count: int, + check_raw_assets: bool, +) -> dict: + """Validate package integrity without invoking AidLite or DSP. + + Dry-run performs three checks: + 1. All four QNN248 context files exist and match the expected SHA256. + 2. The selected frame range exists in the sample manifest. + 3. When ``--check_raw_assets`` is enabled, every referenced raw asset exists. + + This mode is useful on Windows or a normal development container where the + AidLite module is not available. + """ + + # Four model contexts are required by the split BEVFormer pipeline: + # backbone, scene-start encoder, temporal encoder, and decoder. + models = { + "backbone": backbone_model, + "encoder_temporal": encoder_model, + "encoder_scene_start": scene_start_encoder_model, + "decoder": decoder_model, + } + + model_platform = BevFormerModel.resolve_model_platform(models) + expected_sha256 = MODEL_SHA256[model_platform] + + model_records = {} + for name in MODEL_NAMES: + model_path = Path(require_file(f"{name}_model", models[name])) + actual_sha = sha256_file(model_path) + expected_sha = expected_sha256[name] + sha_status = "PASS" if actual_sha == expected_sha else "FAIL" + if sha_status != "PASS": + raise RuntimeError( + f"{name} context SHA mismatch for {model_platform}: " + f"expected={expected_sha} actual={actual_sha}" + ) + print(f"{name.upper()}_CONTEXT_SHA_GATE={sha_status} platform={model_platform} {model_path.name}") + + # Store the expected tensor contract in the dry-run summary so the + # package can be audited without loading AidLite. + model_records[name] = { + "platform": model_platform, + "path": str(model_path), + "sha256": actual_sha, + "expected_sha256": expected_sha, + "expected_tensors": EXPECTED_TENSORS[name], + } + + manifest_path = Path(require_file("asset_manifest", asset_manifest)) + nms_path = Path(require_file("nms_contract", nms_contract)) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + total_frames = int(manifest.get("total_frames", len(manifest["frames"]))) + end = min(total_frames, frame_start + frame_count) + if frame_start < 0 or frame_start >= total_frames or end < frame_start: + raise ValueError(f"Invalid frame range: start={frame_start} count={frame_count} total={total_frames}") + + frames = {} + scene_start_count = 0 + temporal_count = 0 + missing_assets = [] + + for frame_index in range(frame_start, end): + sample = f"sample_{frame_index:03d}" + frame = manifest["frames"][sample] + is_scene_start = bool(frame.get("is_scene_start", False)) + encoder_name = "encoder_scene_start" if is_scene_start else "encoder_temporal" + + # frame000 is a scene-start frame. Later frames use temporal encoder and + # depend on the previous frame's live bev_embed. + if is_scene_start: + scene_start_count += 1 + else: + temporal_count += 1 + + # Raw asset checking is optional because it touches every camera JPG and + # auxiliary tensor path. It is recommended before packaging or upload. + if check_raw_assets: + for asset_name, record in frame.get("assets", {}).items(): + if asset_name == "camera_images": + for image_record in record.get("images", []): + asset_path = _resolve_repo_path(image_record["path"]) + if not asset_path.is_file(): + missing_assets.append({ + "frame": sample, + "asset": f"camera_images/{image_record.get('name', 'UNKNOWN')}", + "path": str(asset_path), + }) + continue + + asset_path = _resolve_repo_path(record["path"]) + if not asset_path.is_file(): + missing_assets.append({ + "frame": sample, + "asset": asset_name, + "path": str(asset_path), + }) + + frames[sample] = { + "sample_token": frame.get("sample_token"), + "is_scene_start": is_scene_start, + "encoder": encoder_name, + "status": "DRY_RUN_PASS", + } + print(f"FRAME {frame_index:03d} DRY_RUN encoder={encoder_name}") + + if missing_assets: + first = missing_assets[0] + raise FileNotFoundError(f"Missing raw asset: {first['frame']} {first['asset']} {first['path']}") + + output_dir.mkdir(parents=True, exist_ok=True) + + run_finished_at = local_timestamp() + + # Write a machine-readable summary. This is useful for proving that the + # package is self-consistent before running on the board. + result = { + "status": "DRY_RUN_PASS", + "run_timestamps": {"finished_at": run_finished_at}, + "note": "AidLite/DSP was not invoked. Run without --dry_run on the board for real inference.", + "manifest": str(manifest_path), + "nms_contract": str(nms_path), + "repo_root": str(REPO_ROOT), + "frame_range": [int(frame_start), int(end - 1)] if end > frame_start else [], + "completed_frames": int(end - frame_start), + "scene_start_encoder_count": scene_start_count, + "temporal_encoder_count": temporal_count, + "model_platform": model_platform, + "models": model_records, + "raw_asset_existence_checked": bool(check_raw_assets), + "frames": frames, + } + + result_path = output_dir / "bevformer_demo_dry_run_summary.json" + result_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") + + print("====================================") + print("BEVFormer demo status: DRY_RUN_PASS") + print(f"model_platform: {model_platform}") + print(f"frames: {result['completed_frames']}") + print(f"scene_start_encoder: {scene_start_count}") + print(f"temporal_encoder: {temporal_count}") + print(f"summary: {result_path}") + print("AidLite/DSP not invoked in dry-run mode.") + print("====================================") + return result + + +def _fmt_ms(value) -> str: + """Format optional millisecond values for console output.""" + + if value is None: + return "N/A" + return f"{float(value):.3f}" + + +def local_timestamp() -> str: + """Return a local ISO-8601 timestamp for run logs and summaries.""" + + return datetime.now().astimezone().isoformat(timespec="seconds") + + +def timestamp_for_filename(value: str) -> str: + """Convert an ISO timestamp to a filesystem-friendly suffix.""" + + return value.replace("-", "").replace(":", "").replace("T", "_").split("+")[0] + + +def timestamp_for_output_dir(value: str) -> str: + """Convert an ISO timestamp to outputs/YYYY_MM_DD_HH_MM style.""" + + dt = datetime.fromisoformat(value) + return dt.strftime("%Y_%m_%d_%H_%M") + + +def make_output_dir(base_dir: Path, started_at: str) -> Path: + """Create a timestamped output directory, avoiding same-minute overwrite.""" + + base_dir = base_dir.expanduser().resolve() + candidate = base_dir / timestamp_for_output_dir(started_at) + if not candidate.exists(): + return candidate + for index in range(2, 100): + numbered = base_dir / f"{candidate.name}_{index:02d}" + if not numbered.exists(): + return numbered + raise RuntimeError(f"Too many output directories already exist for {candidate.name}") + + +def write_run_log(output_dir: Path, result: dict, result_path: Path, command: list[str]) -> dict[str, str]: + """Write a compact human-readable run log with timestamps.""" + + timestamps = result.get("run_timestamps", {}) + started_at = timestamps.get("started_at", "unknown") + finished_at = timestamps.get("finished_at", "unknown") + stamp = timestamp_for_filename(started_at) if started_at != "unknown" else "unknown" + timestamped_log = output_dir / f"run_{stamp}.log" + latest_log = output_dir / "run.log" + e2e = result.get("end_to_end_timing_ms", {}) + qnn = result.get("qnn_invoke_ms", {}) + per_bin = result.get("per_bin_qnn_invoke_ms", {}) + app = result.get("application_timing_ms", {}) + lines = [ + "BEVFormer W8A8 board demo run log", + "========================================", + f"started_at : {started_at}", + f"finished_at : {finished_at}", + f"status : {result.get('status')}", + f"model platform : {result.get('model_platform', 'unknown')}", + f"command : {' '.join(command)}", + f"frames : {result.get('completed_frames')}", + f"scene-start frames : {result.get('scene_start_encoder_count')}", + f"temporal frames : {result.get('temporal_encoder_count')}", + f"mean QNN execute, selected pipeline (ms): {_fmt_ms(qnn.get('mean'))}", + "per-bin QNN invoke only, mean ms :", + f" backbone_context.bin : {_fmt_ms(per_bin.get('backbone_context.bin', {}).get('mean'))}", + f" scene_start_encoder_context.bin : {_fmt_ms(per_bin.get('scene_start_encoder_context.bin', {}).get('mean'))}", + f" temporal_encoder_context.bin : {_fmt_ms(per_bin.get('temporal_encoder_context.bin', {}).get('mean'))}", + f" decoder_context.bin : {_fmt_ms(per_bin.get('decoder_context.bin', {}).get('mean'))}", + f"full inference chain, no drawing (ms): {_fmt_ms(e2e.get('complete_inference_no_visualization_ms'))}", + f"full demo chain, with drawing (ms) : {_fmt_ms(e2e.get('complete_inference_with_visualization_ms'))}", + f"whole Python run incl. load (ms) : {_fmt_ms(app.get('total_until_program_end_ms'))}", + f"summary JSON : {result_path}", + f"output directory : {output_dir}", + ] + if result.get("camera_grid_gif"): + lines.append(f"camera-grid GIF : {result['camera_grid_gif']['path']}") + text = "\n".join(lines) + "\n" + timestamped_log.write_text(text, encoding="utf-8") + latest_log.write_text(text, encoding="utf-8") + return {"timestamped": str(timestamped_log), "latest": str(latest_log)} + + +def main() -> int: + """Run the board demo. + + The function first resolves all paths. If ``--dry_run`` is enabled, it + stops after package checks. Otherwise it loads AidLite contexts through + ``BevFormerModel`` and runs the selected continuous frame span. + """ + + app_start = time.perf_counter_ns() + run_started_at = local_timestamp() + + # 1. Parse CLI and resolve all default paths from demo_config.json. + args = parse_args() + config = load_config(args.config) + + model_platform, model_paths = resolve_model_paths(args, config) + backbone_model = model_paths["backbone"] + encoder_model = model_paths["encoder_temporal"] + scene_start_encoder_model = model_paths["encoder_scene_start"] + decoder_model = model_paths["decoder"] + asset_manifest = args.asset_manifest or demo_path(config, ("inputs", "asset_manifest"), DEFAULT_MANIFEST) + nms_contract = args.nms_contract or demo_path(config, ("postprocess", "nms_contract"), DEFAULT_NMS_CONTRACT) + if args.output_dir: + output_dir = Path(args.output_dir).expanduser().resolve() + else: + output_root = Path(demo_path(config, ("outputs", "default_dir"), DEFAULT_OUTPUT)) + output_dir = make_output_dir(output_root, run_started_at) + + # ``--invoke_nums`` takes precedence over ``--frame_count``. + frame_count = args.invoke_nums if args.invoke_nums is not None else args.frame_count + + # 2. Host-side package inspection path. No AidLite import or DSP execution. + if args.dry_run: + _dry_run( + backbone_model=backbone_model, + encoder_model=encoder_model, + scene_start_encoder_model=scene_start_encoder_model, + decoder_model=decoder_model, + asset_manifest=asset_manifest, + nms_contract=nms_contract, + output_dir=output_dir, + frame_start=args.frame_start, + frame_count=frame_count, + check_raw_assets=args.check_raw_assets, + ) + return 0 + + # 3. Board-side real inference path. This requires ``import aidlite`` to + # succeed inside BevFormerModel. + model_load_start = time.perf_counter_ns() + model = BevFormerModel( + backbone_model=require_file("backbone_model", backbone_model), + encoder_temporal_model=require_file("encoder_model", encoder_model), + encoder_scene_start_model=require_file("scene_start_encoder_model", scene_start_encoder_model), + decoder_model=require_file("decoder_model", decoder_model), + model_type=args.model_type, + ) + model_load_wall_ms = (time.perf_counter_ns() - model_load_start) / 1.0e6 + + # 4. Run sample4: raw JPG preprocessing -> backbone -> encoder -> decoder + # -> NumPy NMSFreeCoder -> NPZ/PNG/GIF outputs. + inference_start = time.perf_counter_ns() + result = model.run_manifest( + manifest_path=require_file("asset_manifest", asset_manifest), + repo_root=REPO_ROOT, + output_dir=output_dir, + nms_contract_path=require_file("nms_contract", nms_contract), + frame_start=args.frame_start, + frame_count=frame_count, + save_all_raw=args.save_all_raw, + visualize=not args.no_visualize, + vis_score_thr=args.vis_score_thr, + vis_max_boxes=args.vis_max_boxes, + check_image_sha=args.check_image_sha, + preprocess_workers=DEFAULT_PREPROCESS_WORKERS, + ) + inference_wall_ms = (time.perf_counter_ns() - inference_start) / 1.0e6 + + # 5. Add application-level timing that is not part of model-only QNN invoke. + result["application_timing_ms"] = { + "model_load_wall_ms": model_load_wall_ms, + "run_manifest_wall_ms": inference_wall_ms, + "total_until_summary_write_excluded_ms": (time.perf_counter_ns() - app_start) / 1.0e6, + } + + result["run_timestamps"] = { + "started_at": run_started_at, + "summary_started_at": local_timestamp(), + } + result_path = output_dir / "bevformer_demo_summary.json" + + # Write once to measure summary serialization overhead, then write again + # after adding the measured value to the summary. + summary_write_start = time.perf_counter_ns() + result_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") + summary_write_ms = (time.perf_counter_ns() - summary_write_start) / 1.0e6 + result["application_timing_ms"]["summary_write_ms"] = summary_write_ms + result["application_timing_ms"]["total_until_program_end_ms"] = (time.perf_counter_ns() - app_start) / 1.0e6 + result["run_timestamps"]["finished_at"] = local_timestamp() + result_path.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") + + # 6. Console summary. + qnn = result.get("qnn_invoke_ms", {}) + per_bin = result.get("per_bin_qnn_invoke_ms", {}) + + print("") + print("========================================") + print(f"BEVFormer W8A8 {model_platform} Performance") + print("========================================") + print(f"model platform : {result.get('model_platform', model_platform)}") + print(f"frames : {result['completed_frames']}") + print(f"QNN invoke (ms) : mean={_fmt_ms(qnn.get('mean'))} min={_fmt_ms(qnn.get('min'))} max={_fmt_ms(qnn.get('max'))}") + print(f" backbone : {_fmt_ms(per_bin.get('backbone_context.bin', {}).get('mean'))}") + print(f" encoder : {_fmt_ms(per_bin.get('scene_start_encoder_context.bin', {}).get('mean'))} (scene_start) {_fmt_ms(per_bin.get('temporal_encoder_context.bin', {}).get('mean'))} (temporal)") + print(f" decoder : {_fmt_ms(per_bin.get('decoder_context.bin', {}).get('mean'))}") + print("") + print("Output files") + print(f" summary JSON : {result_path}") + if result.get("visualizations"): + print(f" camera-grid PNG: {len(result['visualizations'])} frame(s)") + if result.get("camera_grid_gif"): + print(f" camera-grid GIF: {result['camera_grid_gif']['path']}") + print("========================================") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/targets/QCS8550/FP16/code/python/run_test.py b/targets/QCS8550/FP16/code/python/run_test.py new file mode 100644 index 0000000000000000000000000000000000000000..49a5f08f769d06b41c2336ef0046b5218320c03f --- /dev/null +++ b/targets/QCS8550/FP16/code/python/run_test.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from run_demo import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/targets/QCS8550/FP16/code/python/utils.py b/targets/QCS8550/FP16/code/python/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fab01038b5dc054b9e01292a86b56c05deff3192 --- /dev/null +++ b/targets/QCS8550/FP16/code/python/utils.py @@ -0,0 +1,320 @@ +from __future__ import annotations + +import hashlib +import json +from concurrent.futures import ThreadPoolExecutor +import math +import time +from pathlib import Path +from typing import Any, Iterable + +import numpy as np + + +CAMERA_NAMES = ( + "CAM_FRONT", + "CAM_FRONT_RIGHT", + "CAM_FRONT_LEFT", + "CAM_BACK", + "CAM_BACK_LEFT", + "CAM_BACK_RIGHT", +) + +IMAGE_MEAN = np.asarray([123.675, 116.28, 103.53], dtype=np.float32) +IMAGE_STD = np.asarray([58.395, 57.12, 57.375], dtype=np.float32) + +SHAPES = { + "images": (6, 3, 480, 800), + "images_unpadded": (6, 3, 450, 800), + "img_feat": (6, 256, 15, 25), + "encoder_img_feat": (1, 6, 256, 15, 25), + "can_bus": (1, 18), + "shift": (1, 2), + "lidar2img": (1, 6, 4, 4), + "bev": (1, 2500, 256), + "decoder": (1, 900, 10), +} + +EXPECTED_TENSORS = { + "backbone": { + "inputs": {"images": 6912000}, + "outputs": {"img_feat": 576000}, + }, + "encoder_temporal": { + "inputs": { + "can_bus": 18, + "img_feat": 576000, + "lidar2img": 96, + "shift": 2, + "prev_bev": 640000, + }, + "outputs": {"bev_embed": 640000}, + }, + "encoder_scene_start": { + "inputs": { + "can_bus": 18, + "img_feat": 576000, + "lidar2img": 96, + }, + "outputs": {"bev_embed": 640000}, + }, + "decoder": { + "inputs": {"bev_embed": 640000}, + "outputs": {"cls_scores": 9000, "bbox_preds": 9000}, + }, +} + +ROTATE_CENTER = (100.0, 100.0) + + +def sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def load_json(path: str | Path) -> dict[str, Any]: + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def normalize_rc(value: Any) -> int: + return 0 if value is None else int(value) + + +def resolve_path(path: str | Path, repo_root: str | Path) -> Path: + value = Path(path) + if value.is_absolute(): + return value + return Path(repo_root) / value + + +def load_record(record: dict[str, Any], repo_root: str | Path) -> np.ndarray: + path = resolve_path(record["path"], repo_root) + if not path.is_file(): + raise FileNotFoundError(path) + actual_sha = sha256_file(path) + expected_sha = record.get("sha256") + if expected_sha and actual_sha != expected_sha: + raise RuntimeError(f"SHA mismatch for {path}: expected={expected_sha} actual={actual_sha}") + + dtype = " np.ndarray: + try: + import cv2 + except ModuleNotFoundError as exc: + raise RuntimeError("OpenCV is required for board-side camera image preprocessing.") from exc + + image = cv2.imread(str(path), cv2.IMREAD_UNCHANGED) + if image is None: + raise FileNotFoundError(path) + if image.ndim != 3 or image.shape[2] != 3: + raise ValueError(f"Expected 3-channel camera image: {path} shape={image.shape}") + image = image.astype(np.float32) + image = image[..., ::-1] + image = (image - IMAGE_MEAN) / IMAGE_STD + image = cv2.resize(image, (800, 450), interpolation=cv2.INTER_LINEAR) + return image.transpose(2, 0, 1) + + +def _preprocess_one_camera( + index: int, + camera_name: str, + item: dict[str, Any], + repo_root: str | Path, + check_sha: bool, +) -> tuple[int, np.ndarray, dict[str, Any]]: + path = resolve_path(item["path"], repo_root) + if not path.is_file(): + raise FileNotFoundError(path) + actual_sha = None + expected_sha = item.get("sha256") + if check_sha: + actual_sha = sha256_file(path) + if expected_sha and actual_sha != expected_sha: + raise RuntimeError(f"SHA mismatch for {path}: expected={expected_sha} actual={actual_sha}") + tensor = _load_camera_image(path) + if tensor.shape != SHAPES["images_unpadded"][1:]: + raise ValueError(f"Unexpected preprocessed camera tensor shape: {tensor.shape}") + return index, tensor, { + "name": camera_name, + "path": str(path), + "sha_checked": bool(check_sha), + "sha256": actual_sha or expected_sha, + } + + +def preprocess_camera_images( + record: dict[str, Any], + repo_root: str | Path, + check_sha: bool = False, + num_workers: int = 6, +) -> tuple[np.ndarray, dict[str, Any]]: + images_by_name = {item["name"]: item for item in record["images"]} + camera_order = tuple(record.get("order", CAMERA_NAMES)) + if camera_order != CAMERA_NAMES: + raise RuntimeError(f"Unexpected camera order: {camera_order}") + + tasks = [(index, name, images_by_name[name], repo_root, check_sha) for index, name in enumerate(CAMERA_NAMES)] + if int(num_workers) > 1: + with ThreadPoolExecutor(max_workers=int(num_workers)) as executor: + results = list(executor.map(lambda args: _preprocess_one_camera(*args), tasks)) + else: + results = [_preprocess_one_camera(*args) for args in tasks] + + padded = np.zeros(SHAPES["images"], dtype=np.float32) + checked_images = [None] * len(CAMERA_NAMES) + for index, tensor, image_record in results: + padded[index, :, :450, :] = tensor + checked_images[index] = image_record + + return np.ascontiguousarray(padded, dtype=np.float32), { + "source": "camera_images", + "pipeline": record.get( + "pipeline", + "BGR -> RGB -> normalize(mean/std) -> resize(800,450) -> CHW -> pad to 480 rows", + ), + "camera_order": list(CAMERA_NAMES), + "sha_checked": bool(check_sha), + "num_workers": int(num_workers), + "unpadded_shape": list(SHAPES["images_unpadded"]), + "padded_shape": list(padded.shape), + "reference_preprocessed_sha256": record.get("reference_preprocessed_sha256"), + "images": checked_images, + } + + +def load_backbone_images( + assets: dict[str, Any], + repo_root: str | Path, + check_image_sha: bool = False, + preprocess_workers: int = 6, +) -> tuple[np.ndarray, dict[str, Any]]: + if "camera_images" not in assets: + raise KeyError("camera_images is required; this demo starts from raw camera JPG inputs.") + return preprocess_camera_images( + assets["camera_images"], + repo_root, + check_sha=check_image_sha, + num_workers=preprocess_workers, + ) + + +def as_encoder_img_feat(img_feat: np.ndarray) -> np.ndarray: + native = np.ascontiguousarray(img_feat, dtype=" float: + return (time.perf_counter_ns() - start_ns) / 1.0e6 + + +def stats(values: Iterable[float]) -> dict[str, float]: + items = [float(value) for value in values] + if not items: + return {"count": 0} + mean = float(sum(items) / len(items)) + variance = float(sum((value - mean) ** 2 for value in items) / len(items)) + return { + "count": len(items), + "mean": mean, + "min": float(min(items)), + "max": float(max(items)), + "sum": float(sum(items)), + "var": variance, + } + + +def _torchvision_inverse_affine_matrix(center: tuple[float, float], angle: float) -> list[float]: + rot = math.radians(angle) + a = math.cos(rot) + b = -math.sin(rot) + c = math.sin(rot) + d = math.cos(rot) + matrix = [d, -b, 0.0, -c, a, 0.0] + cx, cy = center + matrix[2] += matrix[0] * (-cx) + matrix[1] * (-cy) + matrix[5] += matrix[3] * (-cx) + matrix[4] * (-cy) + matrix[2] += cx + matrix[5] += cy + return matrix + + +def rotate_prev_bev_like_torchvision( + previous_bev: np.ndarray, + angle: float, + rotate_center: tuple[float, float] = ROTATE_CENTER, +) -> np.ndarray: + bev = np.ascontiguousarray(previous_bev, dtype=np.float32).reshape(SHAPES["bev"]) + grid = bev[0].reshape(50, 50, 256) + height, width, channels = grid.shape + center_f = ( + float(rotate_center[0]) - width * 0.5, + float(rotate_center[1]) - height * 0.5, + ) + theta = np.asarray( + _torchvision_inverse_affine_matrix(center_f, -float(angle)), + dtype=np.float32, + ).reshape(2, 3) + x_base = np.linspace(-width * 0.5 + 0.5, width * 0.5 - 0.5, width, dtype=np.float32) + y_base = np.linspace(-height * 0.5 + 0.5, height * 0.5 - 0.5, height, dtype=np.float32) + x_grid, y_grid = np.meshgrid(x_base, y_base) + src_x = (x_grid * theta[0, 0] + y_grid * theta[0, 1] + theta[0, 2]) / (0.5 * width) + src_y = (x_grid * theta[1, 0] + y_grid * theta[1, 1] + theta[1, 2]) / (0.5 * height) + ix = np.rint(((src_x + 1.0) * width - 1.0) * 0.5).astype(np.int64) + iy = np.rint(((src_y + 1.0) * height - 1.0) * 0.5).astype(np.int64) + valid = (ix >= 0) & (ix < width) & (iy >= 0) & (iy < height) + out = np.zeros((height, width, channels), dtype=np.float32) + out[valid] = grid[iy[valid], ix[valid]] + return np.ascontiguousarray(out.reshape(SHAPES["bev"]), dtype=np.float32) + + +def rotate_prev_bev(previous_live_bev: np.ndarray, rotation_can_bus: np.ndarray) -> np.ndarray: + angle = float(rotation_can_bus.reshape(-1)[-1]) + previous_native = np.ascontiguousarray(previous_live_bev, dtype=" dict[str, Any]: + output_dir.mkdir(parents=True, exist_ok=True) + cls_path = output_dir / f"frame{frame_index:03d}_cls_scores_fp32.raw" + bbox_path = output_dir / f"frame{frame_index:03d}_bbox_preds_fp32.raw" + np.ascontiguousarray(cls_scores, dtype=" dict[str, Any]: + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / f"frame{frame_index:03d}_final_coordinates.npz" + np.savez( + path, + boxes=np.ascontiguousarray(boxes, dtype=np.float32), + scores=np.ascontiguousarray(scores, dtype=np.float32), + labels=np.ascontiguousarray(labels, dtype=np.int64), + ) + return { + "path": str(path), + "sha256": sha256_file(path), + "box_count": int(len(scores)), + } diff --git a/targets/QCS8550/FP16/composite.json b/targets/QCS8550/FP16/composite.json new file mode 100644 index 0000000000000000000000000000000000000000..1835d8b54ac6f37de5c04fa3a886954fa7efbebb --- /dev/null +++ b/targets/QCS8550/FP16/composite.json @@ -0,0 +1,199 @@ +{ + "assets": { + "nmsContract": "code/python/configs/nms_runtime_contract.json" + }, + "components": { + "backbone": "models/QCS8550/FP16/backbone_context.bin.aidem", + "decoder": "models/QCS8550/FP16/decoder_context.bin.aidem", + "scene_start": "models/QCS8550/FP16/scene_start_encoder_context.bin.aidem", + "temporal": "models/QCS8550/FP16/temporal_encoder_context.bin.aidem" + }, + "derivation": { + "files": { + "calibration/sample_000.json": "35def1b2da02f8d1b3f5b5dcbb40fb74ddd5074f6dc16f9dd2398509d0d0e6d0", + "calibration/sample_001.json": "d37b43abb24770364bc0f344d1208e5256de9a3f85c5f9b6e3f6cf18db1d094b", + "calibration/sample_002.json": "89c891d43cc43e3665e32fec157b861d12d8fd38324a4ae7a5a8350290ef7c83", + "calibration/sample_003.json": "80052246ff0d259bbcb0b719a0a0c86909808a6f993b94999deef26085ef0920" + }, + "graphHashes": { + "backbone": "9a7a5137a2ed3072cdc2c567822a2316b9501a591ffa204b17568f6c80d8844e", + "decoder": "f6888090751bf1ff6ecbaef966b3b1a6c97793784f921001b493893a8ce6196c", + "scene_start": "eed541fa4138a177432e2290b4131799d88534005789bef9962e2bbb9e008186", + "temporal": "25f0063fc7d1b125522d3ac9c53b26a9e270d504fc6c908ae5737ec78ad65b94" + }, + "precisionLabelNote": "MMS path labels FP16; original demo calls this exact four-graph release W8A8. Preserve vendor label and disclose discrepancy; internal precision not independently established.", + "sourcePackageSha256": "c7c363dd8792dad36b768f768973bd73c63b08c5116aa54955f364340396116c" + }, + "frames": [ + { + "calibration": "calibration/sample_000.json", + "images": [ + { + "bytes": 141976, + "name": "CAM_FRONT", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT.jpg", + "sha256": "0e2067a1618c4424050c45c393ada8c9dbf130fff8d6015004fce901dc07984d" + }, + { + "bytes": 147910, + "name": "CAM_FRONT_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_RIGHT.jpg", + "sha256": "15cf59c79b086d14da6fae5a7ee5b0f4d0fac0c0a22c0a85faaa5fb215521191" + }, + { + "bytes": 144177, + "name": "CAM_FRONT_LEFT", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_FRONT_LEFT.jpg", + "sha256": "8c52f73a3ef06a129c27ef51ac65704428caebfaa60c52f14a5953bbb0124a18" + }, + { + "bytes": 108796, + "name": "CAM_BACK", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK.jpg", + "sha256": "047dd43f3f9fb7cc426b1d4ae369181242c5555b84baff5d7e1cbd0dc8b07259" + }, + { + "bytes": 110941, + "name": "CAM_BACK_LEFT", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_LEFT.jpg", + "sha256": "27b2826991733662e4f17c509ad5a66e85e0ca66a6413063d8db9645fa608c11" + }, + { + "bytes": 124547, + "name": "CAM_BACK_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_000/cameras/CAM_BACK_RIGHT.jpg", + "sha256": "a998caf0e327816b82d2a142d2581a8c1f9c4e67d50b4af3718a381337b7e415" + } + ], + "key": "sample_000" + }, + { + "calibration": "calibration/sample_001.json", + "images": [ + { + "bytes": 146297, + "name": "CAM_FRONT", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT.jpg", + "sha256": "d5624fcecf12310c3d0760362e097c38cdcdce987e032aaf2883071cd14e2732" + }, + { + "bytes": 143628, + "name": "CAM_FRONT_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_RIGHT.jpg", + "sha256": "dd64d7463bc06bfd5d5378bdd494b40d68d0894307f3ed971eaaaf131022cf68" + }, + { + "bytes": 146244, + "name": "CAM_FRONT_LEFT", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_FRONT_LEFT.jpg", + "sha256": "0fe15c0581db41e084df27acb2183b0ab9b0fd36da136ef1b4bcbda6f1a84c58" + }, + { + "bytes": 109601, + "name": "CAM_BACK", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK.jpg", + "sha256": "fdbdaa77e60ed5a6a762a4ea9a8eb1cb2ca9a3d94c610a55a2399a05c04b6ba6" + }, + { + "bytes": 112425, + "name": "CAM_BACK_LEFT", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_LEFT.jpg", + "sha256": "000f7943769e91ffa842a7dfd1bfc31e0c3f4b37325a98846e8261304028f2ad" + }, + { + "bytes": 125712, + "name": "CAM_BACK_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_001/cameras/CAM_BACK_RIGHT.jpg", + "sha256": "39d28a8a5a6f61370a7e7e1e965a6d8610b571195c50fd29da960e1fb2fbee80" + } + ], + "key": "sample_001" + }, + { + "calibration": "calibration/sample_002.json", + "images": [ + { + "bytes": 145856, + "name": "CAM_FRONT", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT.jpg", + "sha256": "0bb82b89f9e6ce47d9e1bcd747b01fe3fd342ccc67eb7a9c693a3fc3fc73a6bc" + }, + { + "bytes": 132652, + "name": "CAM_FRONT_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_RIGHT.jpg", + "sha256": "70856b9d713c279b4b3d5911be30944f339760cc81da5a8f2c4d90a20276c1d3" + }, + { + "bytes": 145095, + "name": "CAM_FRONT_LEFT", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_FRONT_LEFT.jpg", + "sha256": "4782aaa9180ef1fe153e702031a3249ada09bdb056ad13ff69abaa2f9e59e35a" + }, + { + "bytes": 111521, + "name": "CAM_BACK", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK.jpg", + "sha256": "a9a05b52aea16ccf730447170eb836697c26928a112a74249caf423d91d28306" + }, + { + "bytes": 112445, + "name": "CAM_BACK_LEFT", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_LEFT.jpg", + "sha256": "a28438e9ff68ccab2ce3a8da4151591a2b15780aecdc21185a911318ed923f72" + }, + { + "bytes": 132515, + "name": "CAM_BACK_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_002/cameras/CAM_BACK_RIGHT.jpg", + "sha256": "1578ba72afd2cc91abb1295c3404a432e41ffc7385b4031a43b586acfb693872" + } + ], + "key": "sample_002" + }, + { + "calibration": "calibration/sample_003.json", + "images": [ + { + "bytes": 147138, + "name": "CAM_FRONT", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT.jpg", + "sha256": "54384bc8c4896f349bdc643b573f398fbb03ee842ac2af2af2c602a62f48f79d" + }, + { + "bytes": 145853, + "name": "CAM_FRONT_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_RIGHT.jpg", + "sha256": "4c5209d28a7e3a5f29bd7c02c29bd0f53c669afc43b17af2db098371a9711dc1" + }, + { + "bytes": 146337, + "name": "CAM_FRONT_LEFT", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_FRONT_LEFT.jpg", + "sha256": "1f20956c425bdc84da1bf2610b7214d48930652782394aa0f1f3bef94ce0be35" + }, + { + "bytes": 112516, + "name": "CAM_BACK", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK.jpg", + "sha256": "34e84e10e04ab229864bd0ddd3067442c0b58e249eeb48088ab417cb259b4985" + }, + { + "bytes": 113884, + "name": "CAM_BACK_LEFT", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_LEFT.jpg", + "sha256": "cdf1f064c61add58e17d7011713aebbe40be9d7e8290c8816f585fca3841e033" + }, + { + "bytes": 135629, + "name": "CAM_BACK_RIGHT", + "path": "code/python/datasets/sample4/frames/sample_003/cameras/CAM_BACK_RIGHT.jpg", + "sha256": "eae354ba8407662caa6c50934b6ee306a4e5aaaeec0b0f2e40c2f6b481485d3f" + } + ], + "key": "sample_003" + } + ], + "kind": "bevformer", + "version": 1 +} diff --git a/targets/QCS8550/FP16/models/QCS8550/FP16/backbone_context.bin.aidem b/targets/QCS8550/FP16/models/QCS8550/FP16/backbone_context.bin.aidem new file mode 100644 index 0000000000000000000000000000000000000000..28429d8d09f390d194fc8bb0070d97b5eb4da522 --- /dev/null +++ b/targets/QCS8550/FP16/models/QCS8550/FP16/backbone_context.bin.aidem @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a7a5137a2ed3072cdc2c567822a2316b9501a591ffa204b17568f6c80d8844e +size 26945640 diff --git a/targets/QCS8550/FP16/models/QCS8550/FP16/decoder_context.bin.aidem b/targets/QCS8550/FP16/models/QCS8550/FP16/decoder_context.bin.aidem new file mode 100644 index 0000000000000000000000000000000000000000..31756deb31a096b0f8f0a7ea0fca0e4252d2c8f2 --- /dev/null +++ b/targets/QCS8550/FP16/models/QCS8550/FP16/decoder_context.bin.aidem @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6888090751bf1ff6ecbaef966b3b1a6c97793784f921001b493893a8ce6196c +size 18214696 diff --git a/targets/QCS8550/FP16/models/QCS8550/FP16/scene_start_encoder_context.bin.aidem b/targets/QCS8550/FP16/models/QCS8550/FP16/scene_start_encoder_context.bin.aidem new file mode 100644 index 0000000000000000000000000000000000000000..95f49d9a3cef1226efc8eaa7a8e0916e0a47750c --- /dev/null +++ b/targets/QCS8550/FP16/models/QCS8550/FP16/scene_start_encoder_context.bin.aidem @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eed541fa4138a177432e2290b4131799d88534005789bef9962e2bbb9e008186 +size 12364600 diff --git a/targets/QCS8550/FP16/models/QCS8550/FP16/temporal_encoder_context.bin.aidem b/targets/QCS8550/FP16/models/QCS8550/FP16/temporal_encoder_context.bin.aidem new file mode 100644 index 0000000000000000000000000000000000000000..d99175523325ac15c987c595f396696b66df6ffa --- /dev/null +++ b/targets/QCS8550/FP16/models/QCS8550/FP16/temporal_encoder_context.bin.aidem @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25f0063fc7d1b125522d3ac9c53b26a9e270d504fc6c908ae5737ec78ad65b94 +size 12753768 diff --git a/targets/QCS8550/model.json b/targets/QCS8550/model.json new file mode 100644 index 0000000000000000000000000000000000000000..fd44f070eba6720ad7b60921112e2fe46eb15e9d --- /dev/null +++ b/targets/QCS8550/model.json @@ -0,0 +1,7 @@ +{ + "soc": "QCS8550", + "default_precision": "FP16", + "variants": { + "FP16": "FP16/" + } +}