diff --git a/.gitattributes b/.gitattributes index cd4485007a1ed5c2c9a127d0e49e7b3277e41bbd..85a21458d60c4759f8d13c29b6603fe6bd968801 100644 --- a/.gitattributes +++ b/.gitattributes @@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text den_pos_emb.f32 filter=lfs diff=lfs merge=lfs -text assets/hero.png filter=lfs diff=lfs merge=lfs -text +den_joint_w.f32 filter=lfs diff=lfs merge=lfs -text +gs_w.f32 filter=lfs diff=lfs merge=lfs -text diff --git a/262k/bir.tflite b/262k/bir.tflite new file mode 100644 index 0000000000000000000000000000000000000000..8a5574ef0b8ac6a4af7e4f43de29c9444ad5c93f --- /dev/null +++ b/262k/bir.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42b98bf0e497dd596327c31883fec21fe7013a4c9f72d1f67ec8ff4d3f0dfd43 +size 930410016 diff --git a/262k/den_ctx.tflite b/262k/den_ctx.tflite new file mode 100644 index 0000000000000000000000000000000000000000..0f86422edabc86a9383f821d3264582484f29054 --- /dev/null +++ b/262k/den_ctx.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39278b27d00a3dcc70da5e2737f6055d5ba4e7e817423a4cf176aacbfb86230d +size 117711308 diff --git a/262k/den_in.tflite b/262k/den_in.tflite new file mode 100644 index 0000000000000000000000000000000000000000..4ccfa6b3c038a537b71a27d32f54192c9c363df6 --- /dev/null +++ b/262k/den_in.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1350c12da702a4a0abf53cdf528b62a8a75ea4d948c6a4764b264b927b419802 +size 146782524 diff --git a/262k/den_joint_00.tflite b/262k/den_joint_00.tflite new file mode 100644 index 0000000000000000000000000000000000000000..28027b773589ad4aff1f1c60d6068c21ff12017f --- /dev/null +++ b/262k/den_joint_00.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37e269459ef1325bd4a84d1df1545afacf8e33d6d5c232c62f834f343a882f86 +size 56134468 diff --git a/den_joint_01.tflite b/262k/den_joint_01.tflite similarity index 100% rename from den_joint_01.tflite rename to 262k/den_joint_01.tflite diff --git a/den_joint_02.tflite b/262k/den_joint_02.tflite similarity index 100% rename from den_joint_02.tflite rename to 262k/den_joint_02.tflite diff --git a/den_joint_03.tflite b/262k/den_joint_03.tflite similarity index 100% rename from den_joint_03.tflite rename to 262k/den_joint_03.tflite diff --git a/den_joint_04.tflite b/262k/den_joint_04.tflite similarity index 100% rename from den_joint_04.tflite rename to 262k/den_joint_04.tflite diff --git a/den_joint_05.tflite b/262k/den_joint_05.tflite similarity index 100% rename from den_joint_05.tflite rename to 262k/den_joint_05.tflite diff --git a/den_joint_06.tflite b/262k/den_joint_06.tflite similarity index 100% rename from den_joint_06.tflite rename to 262k/den_joint_06.tflite diff --git a/den_joint_07.tflite b/262k/den_joint_07.tflite similarity index 100% rename from den_joint_07.tflite rename to 262k/den_joint_07.tflite diff --git a/den_joint_08.tflite b/262k/den_joint_08.tflite similarity index 100% rename from den_joint_08.tflite rename to 262k/den_joint_08.tflite diff --git a/den_joint_09.tflite b/262k/den_joint_09.tflite similarity index 100% rename from den_joint_09.tflite rename to 262k/den_joint_09.tflite diff --git a/den_joint_10.tflite b/262k/den_joint_10.tflite similarity index 100% rename from den_joint_10.tflite rename to 262k/den_joint_10.tflite diff --git a/den_joint_11.tflite b/262k/den_joint_11.tflite similarity index 100% rename from den_joint_11.tflite rename to 262k/den_joint_11.tflite diff --git a/den_joint_12.tflite b/262k/den_joint_12.tflite similarity index 100% rename from den_joint_12.tflite rename to 262k/den_joint_12.tflite diff --git a/den_joint_13.tflite b/262k/den_joint_13.tflite similarity index 100% rename from den_joint_13.tflite rename to 262k/den_joint_13.tflite diff --git a/den_joint_14.tflite b/262k/den_joint_14.tflite similarity index 100% rename from den_joint_14.tflite rename to 262k/den_joint_14.tflite diff --git a/den_joint_15.tflite b/262k/den_joint_15.tflite similarity index 100% rename from den_joint_15.tflite rename to 262k/den_joint_15.tflite diff --git a/den_joint_16.tflite b/262k/den_joint_16.tflite similarity index 100% rename from den_joint_16.tflite rename to 262k/den_joint_16.tflite diff --git a/den_joint_17.tflite b/262k/den_joint_17.tflite similarity index 100% rename from den_joint_17.tflite rename to 262k/den_joint_17.tflite diff --git a/den_joint_18.tflite b/262k/den_joint_18.tflite similarity index 100% rename from den_joint_18.tflite rename to 262k/den_joint_18.tflite diff --git a/den_joint_19.tflite b/262k/den_joint_19.tflite similarity index 100% rename from den_joint_19.tflite rename to 262k/den_joint_19.tflite diff --git a/den_joint_20.tflite b/262k/den_joint_20.tflite similarity index 100% rename from den_joint_20.tflite rename to 262k/den_joint_20.tflite diff --git a/den_joint_21.tflite b/262k/den_joint_21.tflite similarity index 100% rename from den_joint_21.tflite rename to 262k/den_joint_21.tflite diff --git a/den_joint_22.tflite b/262k/den_joint_22.tflite similarity index 100% rename from den_joint_22.tflite rename to 262k/den_joint_22.tflite diff --git a/262k/den_joint_23.tflite b/262k/den_joint_23.tflite new file mode 100644 index 0000000000000000000000000000000000000000..983427bc856e83444bd2fb3596675f66c7b54121 --- /dev/null +++ b/262k/den_joint_23.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53420d4ff17eeee40d89480fecc549d578b8877a386821eb76c2754a4737ebdb +size 56235684 diff --git a/262k/den_pos_emb.f32 b/262k/den_pos_emb.f32 new file mode 100644 index 0000000000000000000000000000000000000000..23a78392960c15c3b9a14ee753635829b8202cbe --- /dev/null +++ b/262k/den_pos_emb.f32 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1dd2858023be2f47f573ee856d4f0ed2328d47fe0f88d3984038a1b1ffeabb2 +size 33554432 diff --git a/262k/gs_0.tflite b/262k/gs_0.tflite new file mode 100644 index 0000000000000000000000000000000000000000..ea3f9f23aa5a03eadd5cbb2e507cd351dcbcf173 --- /dev/null +++ b/262k/gs_0.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:244ae2ffc6d510c8eab99d2381cb20edb8a7c1715272f7a7efeba5fcd77cfa39 +size 122497768 diff --git a/gs_1.tflite b/262k/gs_1.tflite similarity index 100% rename from gs_1.tflite rename to 262k/gs_1.tflite diff --git a/gs_2.tflite b/262k/gs_2.tflite similarity index 100% rename from gs_2.tflite rename to 262k/gs_2.tflite diff --git a/gs_3.tflite b/262k/gs_3.tflite similarity index 100% rename from gs_3.tflite rename to 262k/gs_3.tflite diff --git a/gs_4.tflite b/262k/gs_4.tflite similarity index 100% rename from gs_4.tflite rename to 262k/gs_4.tflite diff --git a/gs_5.tflite b/262k/gs_5.tflite similarity index 100% rename from gs_5.tflite rename to 262k/gs_5.tflite diff --git a/gs_6.tflite b/262k/gs_6.tflite similarity index 100% rename from gs_6.tflite rename to 262k/gs_6.tflite diff --git a/262k/gs_7.tflite b/262k/gs_7.tflite new file mode 100644 index 0000000000000000000000000000000000000000..4012c09aaba15dca3205307ebc26f1e4164bbbec --- /dev/null +++ b/262k/gs_7.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef75898f78654d4d10692b333dc1e12c860ee54138795aece8e2306ba76217b7 +size 120252780 diff --git a/262k/gs_perturbation.f32 b/262k/gs_perturbation.f32 new file mode 100644 index 0000000000000000000000000000000000000000..3bc4e475150c97eee8e5e9a9a7716b95e9e65231 Binary files /dev/null and b/262k/gs_perturbation.f32 differ diff --git a/262k/gs_rep.json b/262k/gs_rep.json new file mode 100644 index 0000000000000000000000000000000000000000..f4ae45e5b6e5009d7e96b62c7ec697487813c6ff --- /dev/null +++ b/262k/gs_rep.json @@ -0,0 +1,17 @@ +{ + "base_offset_scale": -2.970628499984741, + "rep": { + "num_gaussians": 32, + "lr": { + "_xyz": 1.0, + "_features_dc": 1.0, + "_opacity": 1.0, + "_scaling": 1.0, + "_rotation": 0.1 + }, + "perturbe_size": 1.5, + "3d_filter_kernel_size": 0.0009, + "scaling_bias": 0.004, + "opacity_bias": 0.1 + } +} \ No newline at end of file diff --git a/262k/manifest.json b/262k/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..b74530b0b545621e1add6d70c2a39c04ddcb6fc3 --- /dev/null +++ b/262k/manifest.json @@ -0,0 +1,2316 @@ +{ + "matting": { + "default": "bir", + "alternative": "dis" + }, + "graphs": { + "bir": { + "file": "bir.tflite", + "bytes": 930410016, + "accelerator": "cpu", + "precision": "fp32", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 3, + 1024, + 1024 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 1, + 1024, + 1024 + ] + ] + ] + } + }, + "sha256": "42b98bf0e497dd596327c31883fec21fe7013a4c9f72d1f67ec8ff4d3f0dfd43", + "run_peak_bytes": 2798186496 + }, + "dis": { + "file": "dis.tflite", + "bytes": 176223448, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 3, + 1024, + 1024 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 1, + 1024, + 1024 + ] + ] + ] + } + }, + "sha256": "0c3c93b6a2a65e7c69137ec82596944e6bfb97d982c75bab03acb0738dbaa087", + "repo": "litert-community/DIS-ISNet-LiteRT" + }, + "dino_0": { + "file": "dino_0.tflite", + "bytes": 843659604, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 3, + 1024, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 1, + 4101, + 64 + ] + ], + [ + "args_2", + [ + 1, + 1, + 4101, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 4101, + 1280 + ] + ] + ] + } + }, + "sha256": "7b4fc27cde00f794a846766b660445b08cb94e2380ad9c7f58db191edd3b5b50", + "repo": "litert-community/DINOv3-ViT-H16plus-Idol-LiteRT" + }, + "dino_1": { + "file": "dino_1.tflite", + "bytes": 839694588, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 4101, + 1280 + ] + ], + [ + "args_1", + [ + 1, + 1, + 4101, + 64 + ] + ], + [ + "args_2", + [ + 1, + 1, + 4101, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 4101, + 1280 + ] + ] + ] + } + }, + "sha256": "54a96f0a0ac0cb29e3520c303dba10544ab88f67d97160dbae941ea8f09b41be", + "repo": "litert-community/DINOv3-ViT-H16plus-Idol-LiteRT" + }, + "dino_2": { + "file": "dino_2.tflite", + "bytes": 839694588, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 4101, + 1280 + ] + ], + [ + "args_1", + [ + 1, + 1, + 4101, + 64 + ] + ], + [ + "args_2", + [ + 1, + 1, + 4101, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 4101, + 1280 + ] + ] + ] + } + }, + "sha256": "93b8659dfe738efb3239aa28ff683d503f4caee7b73ebbb9e8914ae27e90f3a1", + "repo": "litert-community/DINOv3-ViT-H16plus-Idol-LiteRT" + }, + "dino_3": { + "file": "dino_3.tflite", + "bytes": 839710040, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 4101, + 1280 + ] + ], + [ + "args_1", + [ + 1, + 1, + 4101, + 64 + ] + ], + [ + "args_2", + [ + 1, + 1, + 4101, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 4101, + 1280 + ] + ] + ] + } + }, + "sha256": "9281d490c988268cd8209f353937677b585e13e6ea324c4f01229f93c89f8c39", + "repo": "litert-community/DINOv3-ViT-H16plus-Idol-LiteRT" + }, + "vae_0": { + "file": "vae_0.tflite", + "bytes": 3036028, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 3, + 1024, + 1024 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 128, + 512, + 512 + ] + ] + ] + } + }, + "sha256": "a8c8b881165717d854bead656620f77bfccc71b5f7772c160753723c5f80c2d7" + }, + "vae_1": { + "file": "vae_1.tflite", + "bytes": 10858720, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 128, + 512, + 512 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 256, + 256, + 256 + ] + ] + ] + } + }, + "sha256": "07e357358a2e50a14462be77433f0c11ae2319d0912d320a65335d71fabfd48b" + }, + "vae_2": { + "file": "vae_2.tflite", + "bytes": 43164896, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 256, + 256, + 256 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 512, + 128, + 128 + ] + ] + ] + } + }, + "sha256": "203175990bbbf43a66263ae69ec8000f19bc943314226bf2e26b3d0b0b26eb3f" + }, + "vae_3": { + "file": "vae_3.tflite", + "bytes": 56792512, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 512, + 128, + 128 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 512, + 128, + 128 + ] + ] + ] + } + }, + "sha256": "d3375079a1558cdd4f4c3f96337830ce81d2f37089c2cd79e701cfc393ecc67e" + }, + "vae_attn": { + "file": "vae_attn.tflite", + "bytes": 4423776, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 512, + 128, + 128 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 512, + 128, + 128 + ] + ] + ] + } + }, + "sha256": "4bec7ec37ad58661abb63ee1e16ccb7646afc6d617acf3caa714721912237d88" + }, + "vae_4": { + "file": "vae_4.tflite", + "bytes": 20179348, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 512, + 128, + 128 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 64, + 128, + 128 + ] + ] + ] + } + }, + "sha256": "6ea00886e1fb2bebbd4be40b6591a04abea2208fd6d59e32a4d52310b0d35853" + }, + "den_ctx": { + "file": "den_ctx.tflite", + "bytes": 117711308, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 4101, + 1280 + ] + ], + [ + "args_1", + [ + 1, + 4101, + 128 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 4101, + 1024 + ] + ] + ] + } + }, + "sha256": "39278b27d00a3dcc70da5e2737f6055d5ba4e7e817423a4cf176aacbfb86230d" + }, + "den_in": { + "file": "den_in.tflite", + "bytes": 146782524, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 16 + ] + ], + [ + "args_1", + [ + 1, + 1, + 5 + ] + ], + [ + "args_2", + [ + 1, + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_4", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 1024 + ] + ], + [ + "output_1", + [ + 1, + 1, + 1024 + ] + ], + [ + "output_2", + [ + 1, + 6144 + ] + ], + [ + "output_3", + [ + 1, + 1024 + ] + ] + ] + } + }, + "sha256": "1350c12da702a4a0abf53cdf528b62a8a75ea4d948c6a4764b264b927b419802" + }, + "den_joint_00": { + "file": "den_joint_00.tflite", + "bytes": 56134468, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 4101, + 1024 + ] + ], + [ + "args_2", + [ + 1, + 1, + 1024 + ] + ], + [ + "args_3", + [ + 1, + 6144 + ] + ], + [ + "args_4", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "37e269459ef1325bd4a84d1df1545afacf8e33d6d5c232c62f834f343a882f86" + }, + "den_joint_01": { + "file": "den_joint_01.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "103efe94ce85aeb29f084f85962d0e85d4a395c629101d3ea21d23e6cdc53e25" + }, + "den_joint_02": { + "file": "den_joint_02.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "1ec688cf610156841a1523db9ed93ad268274f957132dcbc92539308143ce517" + }, + "den_joint_03": { + "file": "den_joint_03.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "b5828ac85b0f42118e79f2e83cbe05021d2b68f9a3c167f0ac014a3026799cc2" + }, + "den_joint_04": { + "file": "den_joint_04.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "ea5c57b89c8d8bbc5144d5d487e20fc49daf25308bccf853bb3c9d4581ae33fe" + }, + "den_joint_05": { + "file": "den_joint_05.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "f339d8b79e5d4c2fc408c2d42f7ce959b81821cad37861b2a1294b8397e19997" + }, + "den_joint_06": { + "file": "den_joint_06.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "30d97e62ac4fc28882903961c88e65e6fe540d61e95504651b44d7cb8f341c4c" + }, + "den_joint_07": { + "file": "den_joint_07.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "0c7b243e55c69b6cdd9c5e7bbef313f6cc6be0c0f7ad32e428187e22b2183a67" + }, + "den_joint_08": { + "file": "den_joint_08.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "0eb17bea8ae04ea0d88d79b327515e1a106bb5339d72440a8ec63712865e3f27" + }, + "den_joint_09": { + "file": "den_joint_09.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "353ece07e6b4888d80ab08e2bdd6dc2dd456f213ca87694f6d77f987d5d4debc" + }, + "den_joint_10": { + "file": "den_joint_10.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "0c7e15340ea4212369ae85f3f9db625f21ccd1747038e0725675c5a7ca32ac07" + }, + "den_joint_11": { + "file": "den_joint_11.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "76cc264ae1ce820749f9dd4758e07fa548ee805a7487cbd4ba342eccc523d779" + }, + "den_joint_12": { + "file": "den_joint_12.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "75ff657c9c7cca82e45ec36fc259cde4cfc90829f09639cdc749d8bddce73f27" + }, + "den_joint_13": { + "file": "den_joint_13.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "9ecae224a9b4962e1e1b1d719942ea89182c77b1f1cf629017f7117d9dded5b2" + }, + "den_joint_14": { + "file": "den_joint_14.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "59f957bdcbcef54f90ac90fc2ed9ff52f40fa93afeb589f6fa377c734ddb4c5c" + }, + "den_joint_15": { + "file": "den_joint_15.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "4839713f18df82b2dd53ed2d605a9d98240bee6bebc00e281a1316246b5ffaaa" + }, + "den_joint_16": { + "file": "den_joint_16.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "a8cc4d07fe9830a8f2fc72420db955c6d08f6093fc82ea9b329df4d3275599e2" + }, + "den_joint_17": { + "file": "den_joint_17.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "29c8966f6b967250b818ebc902b0b1c6070436700c4cd52ad65ab26a3b102387" + }, + "den_joint_18": { + "file": "den_joint_18.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "e5f035dbfa13fcbb92f0212d4c773b8595ae1765b6658a9ecac11dfc80059b21" + }, + "den_joint_19": { + "file": "den_joint_19.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "48394c1fbb680f7bb68ccf3e0253afc170ef47f0f14dc1163d0c84fb9c46222d" + }, + "den_joint_20": { + "file": "den_joint_20.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "2be313d01a72354c1b5dbc291ff1dcecf6345b02e7805e9214c019094762d063" + }, + "den_joint_21": { + "file": "den_joint_21.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "c9c05edb15046307eb06de930d335bdc3b532f5eb9a79314664bb6ecfb087e4e" + }, + "den_joint_22": { + "file": "den_joint_22.tflite", + "bytes": 56134100, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 12294, + 1024 + ] + ] + ] + } + }, + "sha256": "e7674dcda5bef2fca961742e0cb58736e9bed70485a2cad96080bbcddea3490b" + }, + "den_joint_23": { + "file": "den_joint_23.tflite", + "bytes": 56235684, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 12294, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 6144 + ] + ], + [ + "args_2", + [ + 1, + 1024 + ] + ], + [ + "args_3", + [ + 1, + 64 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 16 + ] + ], + [ + "output_1", + [ + 1, + 1, + 5 + ] + ] + ] + } + }, + "sha256": "53420d4ff17eeee40d89480fecc549d578b8877a386821eb76c2754a4737ebdb" + }, + "oct": { + "file": "oct.tflite", + "bytes": 204247600, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "L64": { + "inputs": [ + [ + "args_0", + [ + 1, + 64, + 3 + ] + ], + [ + "args_1", + [ + 1, + 64, + 1024 + ] + ], + [ + "args_2", + [ + 1, + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 64, + 8 + ] + ] + ] + }, + "L256": { + "inputs": [ + [ + "args_0", + [ + 1, + 256, + 3 + ] + ], + [ + "args_1", + [ + 1, + 256, + 1024 + ] + ], + [ + "args_2", + [ + 1, + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 256, + 8 + ] + ] + ] + }, + "L1024": { + "inputs": [ + [ + "args_0", + [ + 1, + 1024, + 3 + ] + ], + [ + "args_1", + [ + 1, + 1024, + 1024 + ] + ], + [ + "args_2", + [ + 1, + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 1024, + 8 + ] + ] + ] + }, + "L4096": { + "inputs": [ + [ + "args_0", + [ + 1, + 4096, + 3 + ] + ], + [ + "args_1", + [ + 1, + 4096, + 1024 + ] + ], + [ + "args_2", + [ + 1, + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 4096, + 8 + ] + ] + ] + }, + "L8192": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 3 + ] + ], + [ + "args_1", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_2", + [ + 1, + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 8 + ] + ] + ] + } + }, + "sha256": "6dab2c2dc79013e798ffabf41dea246bf447cee7bc9336430e8d773162a7cda7" + }, + "gs_0": { + "file": "gs_0.tflite", + "bytes": 122497768, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 3 + ] + ], + [ + "args_1", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_2", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 1024 + ] + ] + ] + } + }, + "sha256": "244ae2ffc6d510c8eab99d2381cb20edb8a7c1715272f7a7efeba5fcd77cfa39" + }, + "gs_1": { + "file": "gs_1.tflite", + "bytes": 118281876, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 1024 + ] + ] + ] + } + }, + "sha256": "08e405e5b545d61b02b9f053d79c09f1300b57cd0f4469ffa65853eb8178bd4d" + }, + "gs_2": { + "file": "gs_2.tflite", + "bytes": 118281876, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 1024 + ] + ] + ] + } + }, + "sha256": "4280e07069d2a6cd271e54477164b8b632d588030e46909a1854a8b35e30eb93" + }, + "gs_3": { + "file": "gs_3.tflite", + "bytes": 118281876, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 1024 + ] + ] + ] + } + }, + "sha256": "5dcd7ecb298a160af606530df20f018b91384cd25e06fe25a0b42401cba7e651" + }, + "gs_4": { + "file": "gs_4.tflite", + "bytes": 118281876, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 1024 + ] + ] + ] + } + }, + "sha256": "37541cee18d0de7b62e104391af8f61eac192424f58a9613cd8281fe6c23f472" + }, + "gs_5": { + "file": "gs_5.tflite", + "bytes": 118281876, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 1024 + ] + ] + ] + } + }, + "sha256": "80972de6ec50f8d6f1ace44c47f076d0f214c0cdb99b6467d37e82af1de940c3" + }, + "gs_6": { + "file": "gs_6.tflite", + "bytes": 118281876, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 1024 + ] + ] + ] + } + }, + "sha256": "50dcc38afa5253d24fb792798db27327905b766404506405ead8e525cb0ec177" + }, + "gs_7": { + "file": "gs_7.tflite", + "bytes": 120252780, + "accelerator": "gpu", + "precision": "fp16_with_fp32_accum", + "signatures": { + "serving_default": { + "inputs": [ + [ + "args_0", + [ + 1, + 8192, + 1024 + ] + ], + [ + "args_1", + [ + 1, + 8192, + 16 + ] + ] + ], + "outputs": [ + [ + "output_0", + [ + 1, + 8192, + 480 + ] + ] + ] + } + }, + "sha256": "ef75898f78654d4d10692b333dc1e12c860ee54138795aece8e2306ba76217b7" + } + }, + "side_files": { + "den_pos_emb.f32": { + "shape": [ + 1, + 8192, + 1024 + ], + "dtype": "float32_le" + }, + "dino_rope_cos.f32": { + "shape": [ + 1, + 1, + 4101, + 64 + ], + "dtype": "float32_le", + "repo": "litert-community/DINOv3-ViT-H16plus-Idol-LiteRT" + }, + "dino_rope_sin.f32": { + "shape": [ + 1, + 1, + 4101, + 64 + ], + "dtype": "float32_le", + "repo": "litert-community/DINOv3-ViT-H16plus-Idol-LiteRT" + }, + "gs_perturbation.f32": { + "shape": [ + 32, + 3 + ], + "dtype": "float32_le" + } + }, + "json_files": [ + "vae_bn.json", + "gs_rep.json" + ], + "total_graph_bytes": 3837368884, + "repos": { + "main": "litert-community/Idol", + "dino": "litert-community/DINOv3-ViT-H16plus-Idol-LiteRT", + "dis": "litert-community/DIS-ISNet-LiteRT" + } +} \ No newline at end of file diff --git a/262k/oct.tflite b/262k/oct.tflite new file mode 100644 index 0000000000000000000000000000000000000000..266ca022d633221e6e8862cdfecb79c2febb535c --- /dev/null +++ b/262k/oct.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dab2c2dc79013e798ffabf41dea246bf447cee7bc9336430e8d773162a7cda7 +size 204247600 diff --git a/262k/vae_0.tflite b/262k/vae_0.tflite new file mode 100644 index 0000000000000000000000000000000000000000..d73280a6d3419d1bd8710bf2f128c595d6fe3a6d --- /dev/null +++ b/262k/vae_0.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8c8b881165717d854bead656620f77bfccc71b5f7772c160753723c5f80c2d7 +size 3036028 diff --git a/262k/vae_1.tflite b/262k/vae_1.tflite new file mode 100644 index 0000000000000000000000000000000000000000..253b03b99a65d3a00621e7b4e21035bccdaf15bd --- /dev/null +++ b/262k/vae_1.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07e357358a2e50a14462be77433f0c11ae2319d0912d320a65335d71fabfd48b +size 10858720 diff --git a/262k/vae_2.tflite b/262k/vae_2.tflite new file mode 100644 index 0000000000000000000000000000000000000000..fac02d60fb437af8ff147fad57e8089e94515480 --- /dev/null +++ b/262k/vae_2.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:203175990bbbf43a66263ae69ec8000f19bc943314226bf2e26b3d0b0b26eb3f +size 43164896 diff --git a/262k/vae_3.tflite b/262k/vae_3.tflite new file mode 100644 index 0000000000000000000000000000000000000000..4e811280b7ade1ea75de71f6fb3cc771415df153 --- /dev/null +++ b/262k/vae_3.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3375079a1558cdd4f4c3f96337830ce81d2f37089c2cd79e701cfc393ecc67e +size 56792512 diff --git a/262k/vae_4.tflite b/262k/vae_4.tflite new file mode 100644 index 0000000000000000000000000000000000000000..6bce41b18c1dee4fc716938cd462cc3c4ddb145b --- /dev/null +++ b/262k/vae_4.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ea00886e1fb2bebbd4be40b6591a04abea2208fd6d59e32a4d52310b0d35853 +size 20179348 diff --git a/262k/vae_attn.tflite b/262k/vae_attn.tflite new file mode 100644 index 0000000000000000000000000000000000000000..f47d9f6622ebfe447d5c2a21087310a7c737f348 --- /dev/null +++ b/262k/vae_attn.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bec7ec37ad58661abb63ee1e16ccb7646afc6d617acf3caa714721912237d88 +size 4423776 diff --git a/262k/vae_bn.json b/262k/vae_bn.json new file mode 100644 index 0000000000000000000000000000000000000000..1ec65bf3a0871d23adc289bc4dd11c237d7f46ab --- /dev/null +++ b/262k/vae_bn.json @@ -0,0 +1 @@ +{"bn_mean": [-0.0673828125, -0.0712890625, -0.0751953125, -0.07470703125, 0.0223388671875, 0.0179443359375, 0.01422119140625, 0.018310546875, -6.29425048828125e-05, -0.006256103515625, -0.000209808349609375, -0.003143310546875, -0.0272216796875, -0.028076171875, -0.027587890625, -0.029052734375, -0.07666015625, -0.0673828125, -0.09033203125, -0.08935546875, 0.016845703125, 0.01519775390625, 0.00787353515625, 0.00860595703125, 0.00836181640625, 0.0015411376953125, 0.0002574920654296875, -0.0042724609375, -0.0439453125, -0.0419921875, -0.043701171875, -0.043212890625, -0.01025390625, -0.01318359375, -0.006622314453125, -0.0047607421875, -0.031005859375, -0.030517578125, -0.0279541015625, -0.0179443359375, 0.003021240234375, 0.00150299072265625, 0.0125732421875, 0.01446533203125, 0.03466796875, 0.03369140625, 0.03369140625, 0.0283203125, 0.001983642578125, 0.004730224609375, 0.004669189453125, 0.004974365234375, 0.01226806640625, 0.00811767578125, 0.008056640625, 0.01458740234375, 0.06787109375, 0.06787109375, 0.07666015625, 0.0732421875, -0.046142578125, -0.04736328125, -0.039306640625, -0.051025390625, -0.052734375, -0.0478515625, -0.047119140625, -0.0517578125, -0.03173828125, -0.03173828125, -0.034423828125, -0.0281982421875, 0.051025390625, 0.04443359375, 0.057861328125, 0.0458984375, -0.041259765625, -0.0458984375, -0.048828125, -0.046630859375, -0.00885009765625, -0.0106201171875, -0.0087890625, -0.004608154296875, -0.03759765625, -0.043212890625, -0.04345703125, -0.0498046875, 0.0118408203125, 0.0166015625, 0.020263671875, 0.0279541015625, 0.01129150390625, 0.01287841796875, 0.001556396484375, 0.00714111328125, -0.01177978515625, -0.00183868408203125, -0.01416015625, -0.00537109375, -0.00909423828125, -0.0137939453125, -0.01446533203125, -0.0186767578125, 0.0322265625, 0.030517578125, 0.02587890625, 0.0299072265625, 0.053955078125, 0.0615234375, 0.049560546875, 0.05908203125, -0.051025390625, -0.060302734375, -0.0478515625, -0.052490234375, -0.022705078125, -0.0274658203125, -0.015380859375, -0.0255126953125, -0.05712890625, -0.056396484375, -0.0517578125, -0.049560546875, 0.0115966796875, 0.00543212890625, 0.016357421875, 0.0103759765625], "bn_var": [3.25, 3.15625, 3.1875, 3.1875, 3.140625, 3.09375, 3.09375, 3.0625, 3.0, 3.015625, 3.0, 3.0, 3.46875, 3.4375, 3.46875, 3.453125, 3.09375, 3.078125, 3.078125, 3.09375, 3.015625, 3.03125, 3.015625, 3.03125, 3.0, 2.96875, 3.03125, 3.0, 3.078125, 3.0625, 3.078125, 3.0625, 3.40625, 3.40625, 3.4375, 3.4375, 3.328125, 3.171875, 3.1875, 3.21875, 3.25, 3.203125, 3.125, 3.125, 3.15625, 3.125, 3.21875, 3.1875, 3.09375, 3.03125, 3.0625, 3.015625, 3.21875, 3.234375, 3.21875, 3.21875, 3.15625, 3.15625, 3.140625, 3.15625, 3.078125, 3.046875, 3.125, 3.0625, 3.15625, 3.140625, 3.171875, 3.171875, 3.296875, 3.25, 3.25, 3.25, 3.078125, 3.0, 3.078125, 3.0625, 3.09375, 3.0625, 3.125, 3.109375, 3.125, 3.078125, 3.171875, 3.140625, 3.203125, 3.234375, 3.1875, 3.15625, 3.109375, 3.109375, 3.078125, 3.09375, 3.15625, 3.125, 3.171875, 3.171875, 2.953125, 2.90625, 2.984375, 2.921875, 3.171875, 3.09375, 3.0625, 3.046875, 3.09375, 3.0625, 3.078125, 3.015625, 3.109375, 3.09375, 3.046875, 3.03125, 3.0625, 3.09375, 3.109375, 3.109375, 2.984375, 2.9375, 3.0, 2.96875, 3.125, 3.109375, 3.140625, 3.125, 3.046875, 3.046875, 3.09375, 3.078125], "bn_eps": 0.0001} \ No newline at end of file diff --git a/README.md b/README.md index 22e6c78b33557e2604a469580d6f3d1f1081ad02..a7d077bc0a094ff26fbe4d1ae39ab4178f5b7635 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,10 @@ tags: # Idol on LiteRT: one photo to a 3D Gaussian splat on the phone -Idol (speridlabs/idol with the idol-turbo LoRA) as LiteRT graphs for `CompiledModel`, converted with litert-torch: on a Galaxy S26, the Android sample in `android/` turns one photo into a 3D Gaussian splat of 262,144 Gaussians in 303 s (the first photo after start on a cool phone, GPU fp16 with fp32 accumulation, LiteRT 2.3.0, measured 2026-11-09). +Idol (speridlabs/idol with the idol-turbo LoRA) as LiteRT graphs for `CompiledModel`, converted with litert-torch: on a Galaxy S26, the Android sample in `android/` turns one photo into a 3D Gaussian splat of 65,536 Gaussians in 158 s with the default form, or of 262,144 Gaussians in 267 s with the first release's form in `262k/` (the first photo after start on a cool phone, LiteRT 2.3.0). -[speridlabs/idol](https://huggingface.co/speridlabs/idol) turns one photo of an object into a 3D Gaussian splat of -262,144 Gaussians that you can turn around, including the sides the picture never showed. This repo holds the +[speridlabs/idol](https://huggingface.co/speridlabs/idol) turns one photo of an object into a 3D Gaussian splat (262,144 Gaussians at its +defaults) that you can turn around, including the sides the picture never showed. This repo holds the author's denoiser with the [speridlabs/idol-turbo](https://huggingface.co/speridlabs/idol-turbo) LoRA (5 steps, no guidance), its octree and Gaussian decoders, the FLUX.2 VAE encoder, and the author's BiRefNet background remover. The DINOv3 image encoder is @@ -28,50 +28,87 @@ in a separate, gated repo under its own licence: ![The photo, and the splat made from it turned to the front, the side and the back](assets/hero.png) -*Left: [the photo](https://www.pexels.com/photo/plant-in-silver-flowerpot-20324601/) by Komet Flicker (Pexels License). Right: the splat the sample app made from it on a Galaxy S26, rendered from the front, the side and the back.* +*Left: [the photo](https://www.pexels.com/photo/plant-in-silver-flowerpot-20324601/) by Komet Flicker (Pexels License). Right: the splat the sample app made from it on a Galaxy S26 with the default form (65,536 Gaussians), rendered from the front, the side and the back.* -## Speed +## Two forms + +The default form (the repo's root) is the fastest of the forms measured on a Galaxy S26: it makes a splat in 158 s against 267 s for the first release's form in `262k/`, while its renders agree less closely with the author's default output: 32.3–34.4 dB PSNR against 41.4–46.6 dB (higher is closer). Of the other forms tried, int8 weights did not run on the phone GPUs, a 512 × 512 input made even the author's own PyTorch pipeline miss the 25 dB bar (13.5–22.9 dB against its 1,024 × 1,024 output), the denoiser did not compile for the Galaxy S26's NPU on the phone, and the CPU alone was slower. + +| | default (the repo's root) | first release (`262k/`) | +|---|---|---| +| Gaussians per splat | 65,536 (2,048 anchors) | 262,144 (8,192 anchors, the author's default) | +| GPU precision | FP16 | FP16 with fp32 accumulation | +| Denoiser blocks 1–22 and Gaussian decoder graphs 1–6 | one shared graph per chain; the app copies each block's weights in before the call | a graph per block | +| Galaxy S26: the first photo after start, cool phone (thermal status 0 at the start) | **158 s** | 267 s | +| Galaxy S26: photos on a phone already warm | 329–351 s (4 photos of the armchair, thermal status 3) | 339–449 s (5 photos of 3 inputs, thermal status 2) | +| Pixel 11 Pro: the first photo after start | 343 s | not measured | +| 4-view render PSNR against the author's default on the Galaxy S26 (threshold 25 dB) | 32.3–34.4 dB | 41.4–46.6 dB | -On a Galaxy S26, the sample app turns one photo into a splat of 262,144 Gaussians in 303 s, starting from a cool phone (the first photo after start). The phone throttles its GPU during that photo: the GPU clock cap fell from 1,300 to 422 MHz within its 5 minutes (thermal status 1 → 3). Photos taken one after another on a phone that was already warm (thermal status 2) took 339–449 s. The graphs do not all fit in the phone's memory at once, so the app closes graphs and compiles them again when the run comes back to them: about 115 compiles per photo, 70 s of the first one. Background removal (BiRefNet) runs on the CPU: 11.1 s of that time on the Galaxy S26. The -other networks run on the GPU. +Thermal status is Android's: 0 none, 1 light, 2 moderate, 3 severe. Both forms run with the same app: point `install_to_device.sh` at the repo's root or at `262k/`. -Speed still being tuned (updates follow). +The first release's card gives 303 s for the `262k/` form: that photo started on a warmer phone (thermal status 1, skin 38.5 °C, against 34.6 °C here). With this app the `262k/` form makes the same splat as with the first release's, byte for byte. + +The rest of this card describes the default form; the first release's card ([commit e7ee1226](https://huggingface.co/litert-community/Idol/blob/e7ee12262a3ed622fa44fb8df8ff4f335dd77064/README.md)) has the +per-stage tables of the `262k/` form. + +## Speed -| Galaxy S26 (SM-S942Q), the sample app, LiteRT 2.3.0; from a cool phone (GPU clock cap 1,300 MHz, thermal status 1 at the start) | first photo after start | +On a Galaxy S26, the sample app turns one photo into a splat of 65,536 Gaussians in 158 s, starting from a cool phone (the first photo after start). The phone throttles its GPU during that photo: the GPU clock cap fell from 1,300 to 500 MHz (thermal status 0 → 2). After 12 minutes of back-to-back runs the phone was at thermal status 3; through the next 4 photos the GPU clock cap stayed at 222–282 MHz (one reading of 191 MHz), and they took 329–351 s. At start the app compiles 1 graph before the photo (2 s) and 19 more during it (20 s): the denoiser's 22 middle blocks share one graph and the Gaussian decoder's six middle graphs share another, so each is compiled once per photo. The first release's form in `262k/` (262,144 Gaussians, a graph per block, GPU FP16 with fp32 accumulation) took 267 s the same way, also from thermal status 0 (skin 34.6 °C at the start against 33.5 °C). Background removal (BiRefNet) runs on the CPU: 7.4 s of that time on the Galaxy S26. The +other networks run on the GPU at FP16. + +| Galaxy S26 (SM-S942Q), the sample app, LiteRT 2.3.0; from a cool phone (GPU clock cap 1,300 MHz, thermal status 0 at the start) | first photo after start | +|---|---| +| **photo → .ply written** | **158 s** | +| background removal (BiRefNet, CPU) | 7.4 s | +| DINOv3 encoder | 10.6 s | +| VAE encoder | 5.2 s | +| denoiser, 5 steps | 119.8 s | +| octree decoder | 9.1 s | +| Gaussian decoder | 5.5 s | +| PLY writer | 0.1 s | +| graphs compiled at app start, before the first photo | 1 (2 s) | +| graphs compiled during the photo | 19 (20 s) | +| peak resident memory (VmHWM) | 5.42 GiB | +| peak GPU memory (kgsl) | 5.73 GiB | + +| Galaxy S26 (SM-S942Q), the sample app, LiteRT 2.3.0; right after the cool photo's hold (thermal status 3 at the start) | first photo after start | photo 2 | photo 3 | photo 4 | +|---|---|---|---|---| +| **photo → .ply written** | **351 s** | **351 s** | **341 s** | **329 s** | +| background removal (BiRefNet, CPU) | 14.9 s | 18.9 s | 15.2 s | 15.4 s | +| DINOv3 encoder | 22.5 s | 23.6 s | 25.1 s | 20.6 s | +| VAE encoder | 10.7 s | 8.1 s | 8.0 s | 7.5 s | +| denoiser, 5 steps | 278.1 s | 280.3 s | 272.6 s | 265.4 s | +| octree decoder | 14.3 s | 11.7 s | 12.1 s | 12.2 s | +| Gaussian decoder | 9.2 s | 7.7 s | 7.6 s | 7.4 s | +| PLY writer | 0.1 s | 0.0 s | 0.0 s | 0.0 s | +| graphs compiled at app start, before the first photo | 3 (9 s) | — | — | — | +| graphs compiled during the photo | 18 (29 s) | 18 (21 s) | 19 (25 s) | 18 (20 s) | +| peak resident memory (VmHWM) | 5.30 GiB | 4.58 GiB | 5.10 GiB | 4.63 GiB | +| peak GPU memory (kgsl) | 5.73 GiB | 5.12 GiB | 5.13 GiB | 5.04 GiB | + +| Pixel 11 Pro, the sample app, LiteRT 2.3.0; thermal status 0 at the start | first photo after start | |---|---| -| **photo → .ply written** | **303 s** | -| background removal (BiRefNet, CPU) | 11.1 s | -| DINOv3 encoder | 6.3 s | -| VAE encoder | 3.7 s | -| denoiser, 5 steps | 228.4 s | -| octree decoder | 16.2 s | -| Gaussian decoder | 36.0 s | +| **photo → .ply written** | **343 s** | +| background removal (BiRefNet, CPU) | 17.4 s | +| DINOv3 encoder | 8.5 s | +| VAE encoder | 36.1 s | +| denoiser, 5 steps | 242.3 s | +| octree decoder | 12.9 s | +| Gaussian decoder | 23.7 s | | PLY writer | 0.2 s | -| graphs compiled during the photo | 115 (70 s) | -| peak resident memory (VmHWM) | 4.71 GiB | -| peak GPU memory (kgsl) | 5.60 GiB | - -| Galaxy S26 (SM-S942Q), the sample app, LiteRT 2.3.0; thermal status 2 (moderate) from the first photo | first photo after start | photo 2 | photo 3 | -|---|---|---|---| -| **photo → .ply written** | **339 s** | **416 s** | **414 s** | -| background removal (BiRefNet, CPU) | 8.5 s | 17.7 s | 15.9 s | -| DINOv3 encoder | 8.8 s | 31.0 s | 26.7 s | -| VAE encoder | 5.4 s | 8.5 s | 7.3 s | -| denoiser, 5 steps | 262.9 s | 306.9 s | 311.9 s | -| octree decoder | 16.0 s | 15.1 s | 15.3 s | -| Gaussian decoder | 35.6 s | 35.7 s | 36.2 s | -| PLY writer | 0.2 s | 0.1 s | 0.1 s | -| graphs compiled during the photo | 115 (78 s) | 114 (97 s) | 116 (93 s) | -| peak resident memory (VmHWM) | 4.91 GiB | 4.16 GiB | 4.09 GiB | -| peak GPU memory (kgsl) | 5.60 GiB | 5.33 GiB | 5.45 GiB | +| graphs compiled at app start, before the first photo | 6 (60 s) | +| graphs compiled during the photo | 15 (75 s) | +| peak resident memory (VmHWM) | 5.96 GiB | ## How close it is to the author's model Each check runs the author's PyTorch pipeline (float32, CPU) and these graphs on the same input with the same random numbers, against thresholds fixed before the first run. The inputs are two images from the author's gallery (`nb-armchair`, `fig-axolotl`) and one photo (a succulent, Pexels). Per part, each graph gets the author's own input -for that part and its output is compared with the author's (cosine; the threshold is 0.99). The whole pipeline is compared -by rendering both splats from four sides with the same renderer (PSNR; the threshold is 25 dB). +for that part and its output is compared with the author's (cosine; the threshold is 0.99); the octree and Gaussian +decoders are compared with the author's model run at this form's 65,536 Gaussians (`--num-gaussians 65536`). The +whole pipeline is compared with the author's default (262,144 Gaussians) by rendering both splats from four sides with +the same renderer (PSNR; the threshold is 25 dB). ### Per-part agreement on the Mac (LiteRT GPU, Metal, fp16): cosine vs the author's model @@ -79,40 +116,40 @@ by rendering both splats from four sides with the same renderer (PSNR; the thres |---|---|---|---|---| | DINOv3 encoder | feature1 | 0.99989 | 0.99994 | 0.99992 | | VAE encoder | latent tokens | 0.99995 | 0.99993 | 0.99996 | -| Denoiser (5 steps) | latent velocity, worst step | 0.99931 | 0.99923 | 0.99931 | -| Octree decoder (8 levels) | child probabilities, worst level | 0.99946 | 0.99937 | 0.99943 | +| Denoiser (5 steps) | latent velocity, worst step | 0.99931 | 0.99924 | 0.99931 | +| Octree decoder (8 levels) | child probabilities, worst level | 0.99948 | 0.99937 | 0.99944 | | Gaussian decoder | xyz | 1.00000 | 1.00000 | 1.00000 | -| | colour (f_dc) | 0.99970 | 0.99963 | 0.99976 | -| | scale | 0.99833 | 0.99837 | 0.99652 | -| | rotation | 0.99891 | 0.99915 | 0.99901 | -| | opacity | 0.99987 | 0.99871 | 0.99988 | +| | colour (f_dc) | 0.99980 | 0.99987 | 0.99937 | +| | scale | 0.99959 | 0.99925 | 0.99225 | +| | rotation | 0.99953 | 0.99919 | 0.99929 | +| | opacity | 0.99995 | 0.99969 | 0.99969 | -### Per-part agreement on a Galaxy S26 (the app; GPU fp16 with fp32 accumulation, BiRefNet on the CPU): cosine vs the author's model +### Per-part agreement on a Galaxy S26 (the app; GPU FP16, BiRefNet on the CPU): cosine vs the author's model | Part | Compared tensor | armchair | |---|---|---| -| DINOv3 encoder | feature1 | 1.00000 | -| VAE encoder | latent tokens | 0.99999 | -| Denoiser (5 steps) | latent velocity, worst step | 0.99999 | -| Octree decoder (8 levels) | child probabilities, worst level | 0.99994 | +| DINOv3 encoder | feature1 | 0.99987 | +| VAE encoder | latent tokens | 0.99991 | +| Denoiser (5 steps) | latent velocity, worst step | 0.99924 | +| Octree decoder (8 levels) | child probabilities, worst level | 0.99943 | | Gaussian decoder | xyz | 1.00000 | -| | colour (f_dc) | 0.99997 | -| | scale | 0.99990 | -| | rotation | 0.99986 | -| | opacity | 0.99993 | +| | colour (f_dc) | 0.99973 | +| | scale | 0.99939 | +| | rotation | 0.99938 | +| | opacity | 0.99996 | -### Per-part agreement on a Pixel 11 Pro (the app; GPU fp16 with fp32 accumulation, BiRefNet on the CPU): cosine vs the author's model +### Per-part agreement on a Pixel 11 Pro (the app; GPU FP16, BiRefNet on the CPU): cosine vs the author's model | Part | Compared tensor | armchair | |---|---|---| -| DINOv3 encoder | feature1 | 0.99998 | -| VAE encoder | latent tokens | 0.99999 | -| Denoiser (5 steps) | latent velocity, worst step | 0.99989 | -| Octree decoder (8 levels) | child probabilities, worst level | 0.99970 | +| DINOv3 encoder | feature1 | 0.99989 | +| VAE encoder | latent tokens | 0.99997 | +| Denoiser (5 steps) | latent velocity, worst step | 0.99930 | +| Octree decoder (8 levels) | child probabilities, worst level | 0.99945 | | Gaussian decoder | xyz | 1.00000 | -| | colour (f_dc) | 0.99989 | -| | scale | 0.99964 | -| | rotation | 0.99976 | +| | colour (f_dc) | 0.99988 | +| | scale | 0.99963 | +| | rotation | 0.99939 | | | opacity | 0.99998 | ### Background removal @@ -129,19 +166,20 @@ by rendering both splats from four sides with the same renderer (PSNR; the thres | Input | Where | PSNR, 4 views pooled (threshold: 25 dB) | per view: 0°, 90°, 180°, 270° | |---|---|---|---| -| armchair | Mac, LiteRT GPU (Metal, fp16) | 35.51 dB | 34.86, 37.92, 34.15, 36.00 | -| armchair | Galaxy S26, the app | 46.59 dB | 46.53, 45.59, 46.41, 48.25 | -| axolotl | Mac, LiteRT GPU (Metal, fp16) | 32.21 dB | 33.85, 29.70, 34.58, 32.43 | -| axolotl | Galaxy S26, the app | 42.34 dB | 40.72, 44.21, 40.59, 46.32 | -| succulent | Mac, LiteRT GPU (Metal, fp16) | 36.38 dB | 35.41, 36.84, 36.14, 37.39 | -| succulent | Galaxy S26, the app | 41.43 dB | 40.30, 41.09, 42.18, 42.51 | +| armchair | Mac, LiteRT GPU (Metal, fp16) | 35.02 dB | 34.23, 36.99, 33.39, 36.48 | +| armchair | Galaxy S26, the app | 34.41 dB | 34.22, 35.85, 33.69, 34.15 | +| armchair | Pixel 11 Pro, the app | 33.58 dB | 32.42, 34.81, 31.90, 36.89 | +| axolotl | Mac, LiteRT GPU (Metal, fp16) | 31.24 dB | 32.84, 28.79, 33.37, 31.52 | +| axolotl | Galaxy S26, the app | 32.26 dB | 33.27, 30.07, 33.71, 33.04 | +| succulent | Mac, LiteRT GPU (Metal, fp16) | 34.85 dB | 34.05, 34.89, 35.03, 35.58 | +| succulent | Galaxy S26, the app | 34.26 dB | 33.47, 34.26, 34.41, 35.05 | ## Minimal usage ### Python: the whole pipeline ```bash -hf download litert-community/Idol --local-dir Idol +hf download litert-community/Idol --exclude "262k/*" --local-dir Idol # the default form; drop --exclude for both cd Idol pip install -r python/requirements.txt python python/run_idol.py photo.jpg --out out --models . # writes out/gaussian.ply @@ -149,7 +187,7 @@ python python/run_idol.py photo.jpg --out out --models . # writes out/gaussian `run_idol.py` downloads the DINOv3 repo (accept its licence on the Hub and `hf auth login` first) and runs every graph with `CompiledModel`, the GPU for all but BiRefNet. The steps between graphs are in `python/idol_runner.py` and -`python/idol_glue.py`. On an M4 Max Mac (Metal GPU) one photo took 64 s and up to 22 GB of memory. +`python/idol_glue.py`. On an M4 Max Mac (Metal GPU) one photo took 53 s and up to 6 GB of memory (the process footprint). ### Python: one graph with `CompiledModel` (background removal) @@ -191,7 +229,7 @@ val alpha = outputs[0].readFloat() // [1024 * 1024] sigmoid map, 1 = the object for (b in inputs + outputs) b.close() bir.close() val gpu = CompiledModel.Options(Accelerator.GPU).apply { // every other graph - gpuOptions = CompiledModel.GpuOptions(precision = CompiledModel.GpuOptions.Precision.FP16_WITH_FP32_ACCUM) + gpuOptions = CompiledModel.GpuOptions(precision = CompiledModel.GpuOptions.Precision.FP16) } ``` @@ -199,12 +237,13 @@ The Android sample app (photo or camera → progress per stage → a splat you c The graphs are too large for the APK; a script copies them into the app's files folder over adb: ```bash -hf download litert-community/Idol --local-dir ~/Downloads/Idol +hf download litert-community/Idol --exclude "262k/*" --local-dir ~/Downloads/Idol # the default form; drop --exclude to get 262k/ too hf download litert-community/DINOv3-ViT-H16plus-Idol-LiteRT --local-dir ~/Downloads/DINOv3-ViT-H16plus-Idol-LiteRT cd ~/Downloads/Idol/android chmod +x gradlew scripts/install_to_device.sh # the Hub does not keep the executable bit ./gradlew :app:installDebug IDOL_DIR=~/Downloads/Idol DINO_DIR=~/Downloads/DINOv3-ViT-H16plus-Idol-LiteRT ./scripts/install_to_device.sh +# the first release's form instead (download without --exclude first): IDOL_DIR=~/Downloads/Idol/262k, same DINO_DIR ``` Background removal (BiRefNet on the CPU) needs about 2,670 MB of free memory for one call. Before compiling it, the @@ -219,13 +258,20 @@ the call. | Background removal (BiRefNet) | `bir.tflite` | CPU | 1 | | DINOv3 ViT-H+/16 image encoder | `dino_0..3.tflite` (DINOv3 repo) | GPU | 4 | | FLUX.2 VAE encoder | `vae_0..3.tflite`, `vae_attn.tflite`, `vae_4.tflite` | GPU | 6 | -| Denoiser with the turbo LoRA | `den_ctx.tflite`, `den_in.tflite`, `den_joint_00..23.tflite` | GPU | 5 steps × 25, plus `den_ctx` once per LoRA phase | +| Denoiser with the turbo LoRA | `den_ctx.tflite`, `den_in.tflite`, `den_joint_00.tflite`, `den_joint_mid.tflite` (blocks 1–22, weights from `den_joint_w.f32`), `den_joint_23.tflite` | GPU | 5 steps × 25, plus `den_ctx` once per LoRA phase | | Octree decoder | `oct.tflite` (5 signatures by row count) | GPU | 8 levels | -| Gaussian decoder | `gs_0..7.tflite` | GPU | 8 | +| Gaussian decoder | `gs_0.tflite`, `gs_mid.tflite` (graphs 1–6, weights from `gs_w.f32`), `gs_7.tflite` | GPU | 8 | | Octree sampling, Gaussian activation, PLY | host code | CPU | — | -`manifest.json` lists every graph with its signatures, accelerator and precision. On Android the GPU graphs run with -`FP16_WITH_FP32_ACCUM` precision. +`262k/` holds the first release's form: `den_joint_01..22.tflite` and `gs_1..6.tflite` in place of the two shared +graphs, the Gaussian decoder at 8,192 anchors, and its own `manifest.json` (GPU precision FP16 with fp32 accumulation); +the files with the same name in both folders are the same bytes, except `gs_0.tflite`, `gs_7.tflite` and +`manifest.json`. + +`manifest.json` lists every graph with its signatures, accelerator and precision; the GPU graphs run with `FP16` +precision. `den_joint_mid` and `gs_mid` are one graph each for blocks of the same shape: before each call the app +copies that block's weights from the side file into the graph's weight inputs (`shared_blocks`, `block_weights` and +`first_weight_input` in `manifest.json`), so each is compiled once per photo. ## Notes @@ -243,7 +289,7 @@ the call. Gaussian decoder's first block; the chunked form loses 0.6 %. Keep it if you re-export. - **Keep each GPU call short on the Galaxy S26.** In the feasibility run the VAE encoder ran as one graph, about 9 s per GPU call; the screen compositor stalled and the process was killed (signal 9) on its fifth call. Every stage here - is split into several graphs for that reason (on the Galaxy S26 every GPU call takes 0.04–1.83 s: 45 graphs, 49 signatures). + is split into several graphs for that reason (with GPU fp16 with fp32 accumulation, every GPU call of the first form took 0.04–1.83 s on the Galaxy S26 (45 graphs, 49 signatures); this form calls the same blocks, or the Gaussian decoder's at a quarter of the anchors, at FP16). - **The denoiser did not compile for the NPU on the phone.** Compiled on the Galaxy S26 for its NPU (Qualcomm HTP; LiteRT 2.3.0, QAIRT 2.50), one denoiser block did not finish in 15 minutes at the default optimization level (O3) and ran out of memory after 28 s at `optimization_level=1` (optimize for prepare); the HTP compiler warned it has diff --git a/android/app/src/main/java/com/idol/BlockWeights.kt b/android/app/src/main/java/com/idol/BlockWeights.kt new file mode 100644 index 0000000000000000000000000000000000000000..81b4737f908577938beb0329ee05c2b51dbbdf98 --- /dev/null +++ b/android/app/src/main/java/com/idol/BlockWeights.kt @@ -0,0 +1,69 @@ +package com.idol + +import java.io.File +import java.io.RandomAccessFile +import java.nio.ByteOrder +import java.nio.FloatBuffer +import java.nio.channels.FileChannel + +/** + * The weights of the blocks one shared graph runs (the denoiser's middle joint blocks in `den_joint_mid`, the Gaussian + * decoder's middle graphs in `gs_mid`): a little-endian float32 side file, block after block, each block's weight + * inputs in input-name order (from args_ on). The graph is compiled once per photo instead of once + * per block; before each call the block's weights are copied from the mapped file into the graph's input buffers. + */ +class BlockWeights(file: File, private val spec: GraphSpec) : AutoCloseable { + private val raf = RandomAccessFile(file, "r") + private val floats: FloatBuffer + + /** For each weight input: its index in the signature's input list, its offset in one block (floats), its size. */ + private val slots: List> + private val perBlock: Int + + /** One array per distinct input size, reused slot after slot (a write copies it into the input buffer). */ + private val scratch: Map + + init { + try { + val sig = spec.signature() + val indexOf = sig.inputs.withIndex().associate { it.value.name to it.index } + val names = sig.inputs.map { it.name }.filter { trailing(it) >= spec.firstWeightInput } + .sortedBy { trailing(it) } + var offset = 0 + slots = names.map { n -> + val i = indexOf.getValue(n) + val size = sig.inputs[i].size + Triple(i, offset, size).also { offset += size } + } + perBlock = offset + val expect = perBlock.toLong() * spec.sharedBlocks.size * 4 + require(file.length() == expect) { "${file.name}: ${file.length()} bytes, expected $expect" } + floats = raf.channel.map(FileChannel.MapMode.READ_ONLY, 0, file.length()) + .order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer() + scratch = slots.map { it.third }.distinct().associateWith { FloatArray(it) } + } catch (e: Throwable) { + raf.close() + throw e + } + } + + /** Writes block [block]'s weights into [g]'s inputs (tagged: a block already in the buffers is not copied again). */ + fun write(g: Graph, block: Int) { + val pos = spec.sharedBlocks.indexOf(block) + require(pos >= 0) { "block $block is not one of ${spec.name}'s shared blocks" } + val base = pos.toLong() * perBlock + for ((k, slot) in slots.withIndex()) { + val (index, offset, size) = slot + val a = scratch.getValue(size) + floats.position((base + offset).toInt()) + floats.get(a, 0, size) + g.write(index, a, "w${block}_$k") + } + } + + override fun close() { + raf.close() + } + + private fun trailing(name: String): Int = name.substringAfterLast('_').toIntOrNull() ?: -1 +} diff --git a/android/app/src/main/java/com/idol/CardSnippet.kt b/android/app/src/main/java/com/idol/CardSnippet.kt index 09e246dad1dc17c411e108f08b47cc2355409780..15b6c9c5b91379b6dd2130e0e1dba5d6d727bc05 100644 --- a/android/app/src/main/java/com/idol/CardSnippet.kt +++ b/android/app/src/main/java/com/idol/CardSnippet.kt @@ -23,7 +23,7 @@ fun cardSnippet(dir: File, image: FloatArray): String { for (b in inputs + outputs) b.close() bir.close() val gpu = CompiledModel.Options(Accelerator.GPU).apply { // every other graph - gpuOptions = CompiledModel.GpuOptions(precision = CompiledModel.GpuOptions.Precision.FP16_WITH_FP32_ACCUM) + gpuOptions = CompiledModel.GpuOptions(precision = CompiledModel.GpuOptions.Precision.FP16) } // ---- end of the card's block runCatching { env.close() } diff --git a/android/app/src/main/java/com/idol/Glue.kt b/android/app/src/main/java/com/idol/Glue.kt index 5c65f3fe207ce2c38c5e2050882efbd768024547..5b1c22084311848933090765a56e43f199475ee8 100644 --- a/android/app/src/main/java/com/idol/Glue.kt +++ b/android/app/src/main/java/com/idol/Glue.kt @@ -21,20 +21,20 @@ import kotlin.math.tanh class Noise( /** [1, 32, 128, 128] normal: the VAE posterior noise. */ val vae: FloatArray, - /** [1, 8192, 16] normal: x at t = 1. */ + /** [1, 8192, 16] normal: x at t = 1 (the denoiser's latent tokens). */ val latent: FloatArray, /** [1, 1, 5] normal: the camera at t = 1. */ val camera: FloatArray, /** [8, 8192] uniform: the octree sampler's per-row offsets. */ val pool: FloatArray, - /** [1, 8192, 3] uniform: the leaf positions. */ + /** [1, 8192, 3] uniform: the leaf positions (a decoder with fewer anchors uses the first rows). */ val leaf: FloatArray, /** "seed " or the fixture it came from. */ val source: String, ) { init { - require(vae.size == 32 * 128 * 128 && latent.size == Glue.NUM_POINTS * Glue.IN_CH && camera.size == Glue.CAM_CH) - require(pool.size == Glue.MAX_LEVEL * Glue.NUM_POINTS && leaf.size == Glue.NUM_POINTS * 3) + require(vae.size == 32 * 128 * 128 && latent.size == Glue.LATENT_TOKENS * Glue.IN_CH && camera.size == Glue.CAM_CH) + require(pool.size == Glue.MAX_LEVEL * Glue.LATENT_TOKENS && leaf.size == Glue.LATENT_TOKENS * 3) } companion object { @@ -47,10 +47,10 @@ class Noise( fun normal(n: Int) = FloatArray(n) { r.nextGaussian().toFloat() } fun uniform(n: Int) = FloatArray(n) { r.nextFloat() } val vae = normal(32 * 128 * 128) - val latent = normal(Glue.NUM_POINTS * Glue.IN_CH) + val latent = normal(Glue.LATENT_TOKENS * Glue.IN_CH) val camera = normal(Glue.CAM_CH) - val pool = uniform(Glue.MAX_LEVEL * Glue.NUM_POINTS) - val leaf = uniform(Glue.NUM_POINTS * 3) + val pool = uniform(Glue.MAX_LEVEL * Glue.LATENT_TOKENS) + val leaf = uniform(Glue.LATENT_TOKENS * 3) return Noise(vae, latent, camera, pool, leaf, "seed $seed") } } @@ -109,6 +109,13 @@ object Glue { const val MAX_LEVEL = 8 const val PER_ANCHOR = 32 const val PHASE_THRESHOLD = 0.5 + /** The denoiser's latent tokens; also the row count of the shared octree noise ([8, 8192], [1, 8192, 3]). */ + const val LATENT_TOKENS = 8192 + + /** + * The author's anchor count (num_gaussians 262,144 / 32). The graphs set the count a run uses: the Gaussian + * decoder's first input is [1, anchors, 3] (a form exported with fewer anchors makes fewer Gaussians). + */ const val NUM_POINTS = 8192 const val IN_CH = 16 const val CAM_CH = 5 @@ -293,7 +300,8 @@ object Glue { /** * octree_descend: idol OctreeProbabilityFixedlenDecoder.sample for one batch. [probsFn] (cells [rows, 3], rows, - * level) -> child probabilities [rows, 8]. pool [8, 8192], leaf [8192, 3] -> anchors [8192, 3] in [0, 1]^3. + * level) -> child probabilities [rows, 8]. pool [8, >= rows], leaf [>= numPoints, 3] -> anchors [numPoints, 3] + * in [0, 1]^3 (each level reads the first rows of its pool row, the leaves are the first numPoints rows). * [cellsOut], when given, receives each level's cells (the parity record). */ fun octreeDescend( @@ -303,6 +311,8 @@ object Glue { numPoints: Int = NUM_POINTS, levels: Int = MAX_LEVEL, ): FloatArray { + val poolRow = pool.size / levels + require(poolRow >= numPoints && leaf.size >= numPoints * 3) { "octree noise is smaller than $numPoints points" } var prev = IntArray(3) var rows = 1 var counts = intArrayOf(numPoints) @@ -310,7 +320,7 @@ object Glue { val res = (1 shl (l - 1)).toFloat() val cells = FloatArray(rows * 3) { (prev[it].toFloat() + 0.5f) / res } val probs = probsFn(cells, rows, l) - val u = FloatArray(rows) { pool[(l - 1) * numPoints + it] } + val u = FloatArray(rows) { pool[(l - 1) * poolRow + it] } val sampled = systematicRows(probs, counts, u, rows) var m = 0 for (v in sampled) if (v > 0) m++ diff --git a/android/app/src/main/java/com/idol/Graphs.kt b/android/app/src/main/java/com/idol/Graphs.kt index aa9b252092f7777f9e6f2587721a95a1a49600d4..f8a897e594a8f2524d85c543c57273d76080dfbb 100644 --- a/android/app/src/main/java/com/idol/Graphs.kt +++ b/android/app/src/main/java/com/idol/Graphs.kt @@ -37,6 +37,15 @@ class Graph(val spec: GraphSpec, private val model: CompiledModel) : AutoCloseab private fun sig(key: String?): SignatureSpec = spec.signature(key) + /** + * The index of input `args_` in signature [key]: signatures list inputs by name, so a graph with more than 10 + * inputs has args_10 before args_2. [n] itself when no input has that name. + */ + fun argIndex(n: Int, key: String? = null): Int { + val i = sig(key).inputs.indexOfFirst { it.name == "args_$n" } + return if (i >= 0) i else n + } + /** Writes [data] to input [index] of signature [key], unless the buffer already holds [tag]. */ fun write(index: Int, data: FloatArray, tag: Any? = null, key: String? = null) { val s = sig(key) diff --git a/android/app/src/main/java/com/idol/IdolEngine.kt b/android/app/src/main/java/com/idol/IdolEngine.kt index b42c2d05e7567cbfd3c606d0c40ee2520477250f..c52487fe7e86f73a78f4b6e6083c067863b9af33 100644 --- a/android/app/src/main/java/com/idol/IdolEngine.kt +++ b/android/app/src/main/java/com/idol/IdolEngine.kt @@ -114,6 +114,7 @@ class IdolEngine(val modelDir: File, val manifest: Manifest, val options: Engine override fun close() { pool.close() + for (w in blockWeights.values) runCatching { w.close() } runCatching { env.close() } } @@ -126,8 +127,29 @@ class IdolEngine(val modelDir: File, val manifest: Manifest, val options: Engine private val dinoNames = chain("dino_") private val vaeNames = chain("vae_") private val jointNames = chain("den_joint_") + private val gsNames = chain("gs_") + /** + * A chain's blocks in run order as (graph, block): a shared graph (`shared_blocks` in the manifest) stands for + * each of its blocks in turn, with that block's weights written before its call; an ordinary graph has block -1. + */ + private fun steps(names: List): List> = names.flatMap { n -> + val spec = manifest.graph(n) + if (spec.sharedBlocks.isEmpty()) listOf(n to -1) else spec.sharedBlocks.map { n to it } + } + private val jointSteps = steps(jointNames) + private val gsSteps = steps(gsNames) + private val blockWeights: Map = (jointNames + gsNames).map { manifest.graph(it) } + .filter { it.sharedBlocks.isNotEmpty() } + .associate { spec -> + val file = requireNotNull(spec.blockWeights) { "${spec.name} has shared_blocks but no block_weights" } + spec.name to BlockWeights(File(modelDir, file), spec) + } + + /** Anchors per run, from the Gaussian decoder's first input [1, anchors, 3] (8,192 = the author's default). */ + val numPoints: Int = manifest.graph(gsNames[0]).signature().inputs[0].shape[1] + /** The order a run asks for graphs in. */ fun runSchedule(): List { val s = ArrayList() @@ -140,10 +162,10 @@ class IdolEngine(val modelDir: File, val manifest: Manifest, val options: Engine if (slow != phase) s.add("den_ctx") phase = slow s.add("den_in") - s.addAll(jointNames) + s.addAll(jointSteps.map { it.first }) } repeat(Glue.MAX_LEVEL) { s.add("oct") } - s.addAll(gsNames) + s.addAll(gsSteps.map { it.first }) return s } @@ -248,7 +270,7 @@ class IdolEngine(val modelDir: File, val manifest: Manifest, val options: Engine val anchors = Glue.octreeDescend({ cells, rows, level -> progress.stage(Stage.OCTREE, "level $level of ${Glue.MAX_LEVEL}", null, false) Glue.softmax8(octreeLogits(cells, rows, level, latent, latentTag), rows) - }, noise.pool, noise.leaf) + }, noise.pool, noise.leaf, numPoints) times["octree"] = since(t) progress.stage(Stage.OCTREE, null, times["octree"], true) if (!options.preload) pool.release("oct") @@ -331,7 +353,7 @@ class IdolEngine(val modelDir: File, val manifest: Manifest, val options: Engine * (the read waits for [from]'s GPU work), or handed over as the buffer itself at [run] with direct handoff. */ private fun link(from: Graph, fromOut: Int, toName: String, toIn: Int = 0): Link { - if (direct) { + if (direct && from.name != toName) { pool.pin(from.name) try { return Link(from, fromOut, graph(toName), toIn, true) @@ -450,7 +472,8 @@ class IdolEngine(val modelDir: File, val manifest: Manifest, val options: Engine val tEmb = din.read(3) val tModTag = nextTag() var prev = din - for ((k, name) in jointNames.withIndex()) { + for ((k, step) in jointSteps.withIndex()) { + val (name, block) = step val l = link(prev, 0, name) val j = l.to when (k) { @@ -460,14 +483,15 @@ class IdolEngine(val modelDir: File, val manifest: Manifest, val options: Engine j.write(3, tMod, tModTag) j.write(4, gate, gateTag) } - jointNames.size - 1 -> { + jointSteps.size - 1 -> { j.write(1, tMod, tModTag) j.write(2, tEmb) j.write(3, gate, gateTag) } else -> { - j.write(1, tMod, tModTag) - j.write(2, gate, gateTag) + j.write(j.argIndex(1), tMod, tModTag) + j.write(j.argIndex(2), gate, gateTag) + if (block >= 0) blockWeights.getValue(name).write(j, block) } } run(l) @@ -493,19 +517,21 @@ class IdolEngine(val modelDir: File, val manifest: Manifest, val options: Engine return g.read(0, key).copyOf(n * 8) } - /** The Gaussian decoder's 8 graphs: anchors [8192, 3] + latent -> features [8192, 480]. */ + /** The Gaussian decoder's 8 steps: anchors [numPoints, 3] + latent -> features [numPoints, 480]. */ private fun gaussianFeatures(anchors: FloatArray, latent: FloatArray, latentTag: Any?): FloatArray { - val pe = Glue.pointPe(anchors, Glue.NUM_POINTS) - var g = graph(gsNames[0]) + val pe = Glue.pointPe(anchors, numPoints) + var g = graph(gsSteps[0].first) g.write(0, anchors) g.write(1, pe) g.write(2, latent, latentTag) g.run() - for (name in gsNames.drop(1)) { + for ((name, block) in gsSteps.drop(1)) { val l = link(g, 0, name) - l.to.write(1, latent, latentTag) + val next = l.to + next.write(next.argIndex(1), latent, latentTag) + if (block >= 0) blockWeights.getValue(name).write(next, block) run(l) - g = l.to + g = next } return g.read(0) } @@ -567,7 +593,7 @@ class IdolEngine(val modelDir: File, val manifest: Manifest, val options: Engine for (i in 0 until timeGrid.size - 1) { progress.stage(Stage.DENOISER, "step ${i + 1} of ${timeGrid.size - 1}", null, false) val (v, vc) = velocity(fx.floats("x_latent_$i"), fx.floats("x_camera_$i"), timeGrid[i], f1, f2, condTag, ctx) - dump("v_latent_$i", v, intArrayOf(1, Glue.NUM_POINTS, Glue.IN_CH)) + dump("v_latent_$i", v, intArrayOf(1, Glue.LATENT_TOKENS, Glue.IN_CH)) dump("v_camera_$i", vc, intArrayOf(1, 1, Glue.CAM_CH)) } seconds.put("denoiser", since(t)) @@ -593,7 +619,7 @@ class IdolEngine(val modelDir: File, val manifest: Manifest, val options: Engine val feats = gaussianFeatures(anchors, latent, latentTag) seconds.put("gaussian_decoder", since(t)) pool.releaseIf { it.startsWith("gs_") } - dump("gs_features", feats, intArrayOf(1, Glue.NUM_POINTS, 480)) + dump("gs_features", feats, intArrayOf(1, numPoints, 480)) val g = Glue.toGaussians(anchors, feats, rep, perturbation) dump("g_xyz", g.xyz, intArrayOf(g.count, 3)) dump("g_features_dc", g.featuresDc, intArrayOf(g.count, 1, 3)) diff --git a/android/app/src/main/java/com/idol/Manifest.kt b/android/app/src/main/java/com/idol/Manifest.kt index 3162857c527294e51142489f1277f566bab19bce..275f3a039b216039abeef5cb980fffb332f88d00 100644 --- a/android/app/src/main/java/com/idol/Manifest.kt +++ b/android/app/src/main/java/com/idol/Manifest.kt @@ -28,6 +28,14 @@ class GraphSpec( * Null when the manifest does not say. */ val runPeakBytes: Long? = null, + /** + * A graph that runs several blocks of one chain with each block's weights as runtime inputs (`shared_blocks`: the + * blocks it stands for, in order; `block_weights`: the side file with their weights; `first_weight_input`: the + * number of inputs before the weights, args_0 .. args_). Empty for an ordinary graph. + */ + val sharedBlocks: List = emptyList(), + val blockWeights: String? = null, + val firstWeightInput: Int = 3, ) { val onCpu: Boolean get() = accelerator == "cpu" @@ -85,6 +93,10 @@ class Manifest( precision = e.optString("precision", "fp16_with_fp32_accum"), signatures = sigs, runPeakBytes = if (e.has("run_peak_bytes")) e.getLong("run_peak_bytes") else null, + sharedBlocks = e.optJSONArray("shared_blocks")?.let { a -> List(a.length()) { a.getInt(it) } } + ?: emptyList(), + blockWeights = e.optString("block_weights", "").ifEmpty { null }, + firstWeightInput = e.optInt("first_weight_input", 3), ) } val side = LinkedHashMap() diff --git a/android/scripts/install_to_device.sh b/android/scripts/install_to_device.sh index 547c3056678dc6ba0d972e83b7b855bad585c39a..98d73472e4ce93b24c46263d5a859f8ba8c9e01e 100644 --- a/android/scripts/install_to_device.sh +++ b/android/scripts/install_to_device.sh @@ -19,6 +19,7 @@ MODULE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # Defaults for local testing: this worktree's export (models/v2 holds every graph except dis.tflite). # For the published release point them at the downloaded repos (see README), e.g. # IDOL_DIR=~/Downloads/Idol DINO_DIR=~/Downloads/DINOv3-ViT-H16plus-Idol-LiteRT DIS_DIR=~/Downloads/DIS-ISNet-LiteRT +# (IDOL_DIR=~/Downloads/Idol/262k for the first release's form: 262,144 Gaussians, same DINO_DIR) IDOL_DIR="${IDOL_DIR:-$MODULE_DIR/models/v2}" DINO_DIR="${DINO_DIR:-$IDOL_DIR}" DIS_DIR="${DIS_DIR:-$HOME/Downloads/DIS-ISNet-LiteRT}" diff --git a/assets/hero.png b/assets/hero.png index b2d79747269c079f3c4db38479b4285eae465eaf..669f2fd5a74ee453da2e43cdc2181f810399bc4c 100644 --- a/assets/hero.png +++ b/assets/hero.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a5d3126f69a3ecee7081a54f3dded85dd3e344d68ae27f02c89673b740a4de69 -size 1385133 +oid sha256:61a6570eacf0da1d9b2aab8a400dc6a013cf78f8e9b6e8b8340483b28c8a6caa +size 1340605 diff --git a/den_joint_mid.tflite b/den_joint_mid.tflite new file mode 100644 index 0000000000000000000000000000000000000000..e8aee3a3fc914b1ad5a8f583160cd33c53679249 --- /dev/null +++ b/den_joint_mid.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4c96ff1c3d9ca92de152978107c5280d4212591608cd19c6bbab8c301a291be +size 116436 diff --git a/den_joint_w.f32 b/den_joint_w.f32 new file mode 100644 index 0000000000000000000000000000000000000000..d461f73627c77ba6289f472b9712451f216ba8a2 --- /dev/null +++ b/den_joint_w.f32 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac835e519ab94dbf37c0a3f3d48b46156e5dd1eb83cfdbdfd9352d476f4016cb +size 1230118912 diff --git a/gs_0.tflite b/gs_0.tflite index ea3f9f23aa5a03eadd5cbb2e507cd351dcbcf173..335a787e0b36823f826cc9b52ea5366f209307fd 100644 --- a/gs_0.tflite +++ b/gs_0.tflite @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:244ae2ffc6d510c8eab99d2381cb20edb8a7c1715272f7a7efeba5fcd77cfa39 -size 122497768 +oid sha256:a0b0471f4d544c24708298a936d14fe40a4983d0b299dd4826605d32942bc8ec +size 122196424 diff --git a/gs_7.tflite b/gs_7.tflite index 4012c09aaba15dca3205307ebc26f1e4164bbbec..2d4e7172bf6a91d3873b796f4d0f45fbe4727e6a 100644 --- a/gs_7.tflite +++ b/gs_7.tflite @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ef75898f78654d4d10692b333dc1e12c860ee54138795aece8e2306ba76217b7 -size 120252780 +oid sha256:9a30c4304de8c6723d3860c139e0f5bad75bf773fe931b3d60bae9a51521592a +size 119951436 diff --git a/gs_mid.tflite b/gs_mid.tflite new file mode 100644 index 0000000000000000000000000000000000000000..8bb85c57b586dfb464a211ebdf4afe4711d4bf92 --- /dev/null +++ b/gs_mid.tflite @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52e7ab4506909972f66ebd771571833a7740e79d17aa79bc59273eec9d846308 +size 92380 diff --git a/gs_w.f32 b/gs_w.f32 new file mode 100644 index 0000000000000000000000000000000000000000..679e16a5738b66bb1a197215be2c8439679359e2 --- /dev/null +++ b/gs_w.f32 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:382847c5bfe20fab7c6afa777ca6ded030419cfce73506ad253e73001c8fa900 +size 707149824 diff --git a/manifest.json b/manifest.json index b74530b0b545621e1add6d70c2a39c04ddcb6fc3..c6ff8f8ec961470628a1c3ba6472834ad5ce69ae 100644 --- a/manifest.json +++ b/manifest.json @@ -35,48 +35,14 @@ ] } }, - "sha256": "42b98bf0e497dd596327c31883fec21fe7013a4c9f72d1f67ec8ff4d3f0dfd43", - "run_peak_bytes": 2798186496 - }, - "dis": { - "file": "dis.tflite", - "bytes": 176223448, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ - [ - "args_0", - [ - 1, - 3, - 1024, - 1024 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ - 1, - 1, - 1024, - 1024 - ] - ] - ] - } - }, - "sha256": "0c3c93b6a2a65e7c69137ec82596944e6bfb97d982c75bab03acb0738dbaa087", - "repo": "litert-community/DIS-ISNet-LiteRT" + "run_peak_bytes": 2798186496, + "sha256": "42b98bf0e497dd596327c31883fec21fe7013a4c9f72d1f67ec8ff4d3f0dfd43" }, "dino_0": { "file": "dino_0.tflite", "bytes": 843659604, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -127,7 +93,7 @@ "file": "dino_1.tflite", "bytes": 839694588, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -177,7 +143,7 @@ "file": "dino_2.tflite", "bytes": 839694588, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -227,7 +193,7 @@ "file": "dino_3.tflite", "bytes": 839710040, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -277,7 +243,7 @@ "file": "vae_0.tflite", "bytes": 3036028, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -310,7 +276,7 @@ "file": "vae_1.tflite", "bytes": 10858720, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -343,7 +309,7 @@ "file": "vae_2.tflite", "bytes": 43164896, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -376,7 +342,7 @@ "file": "vae_3.tflite", "bytes": 56792512, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -409,7 +375,7 @@ "file": "vae_attn.tflite", "bytes": 4423776, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -442,7 +408,7 @@ "file": "vae_4.tflite", "bytes": 20179348, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -475,7 +441,7 @@ "file": "den_ctx.tflite", "bytes": 117711308, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -521,7 +487,7 @@ "file": "den_in.tflite", "bytes": 146782524, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -604,7 +570,7 @@ "file": "den_joint_00.tflite", "bytes": 56134468, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -661,11 +627,11 @@ }, "sha256": "37e269459ef1325bd4a84d1df1545afacf8e33d6d5c232c62f834f343a882f86" }, - "den_joint_01": { - "file": "den_joint_01.tflite", - "bytes": 56134100, + "den_joint_mid": { + "file": "den_joint_mid.tflite", + "bytes": 116436, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -685,93 +651,85 @@ ] ], [ - "args_2", + "args_10", [ 1, - 64 + 1024, + 1024 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_11", [ 1, - 12294, + 1, 1024 ] - ] - ] - } - }, - "sha256": "103efe94ce85aeb29f084f85962d0e85d4a395c629101d3ea21d23e6cdc53e25" - }, - "den_joint_02": { - "file": "den_joint_02.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_12", [ 1, - 12294, + 1024, 1024 ] ], [ - "args_1", + "args_13", [ 1, - 6144 + 1, + 1024 ] ], [ - "args_2", + "args_14", [ 1, + 1, + 16, 64 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_15", [ 1, - 12294, + 1, + 16, + 64 + ] + ], + [ + "args_16", + [ + 1, + 1024, 1024 ] - ] - ] - } - }, - "sha256": "1ec688cf610156841a1523db9ed93ad268274f957132dcbc92539308143ce517" - }, - "den_joint_03": { - "file": "den_joint_03.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_17", [ 1, - 12294, + 1, 1024 ] ], [ - "args_1", + "args_18", [ 1, - 6144 + 1024, + 4096 + ] + ], + [ + "args_19", + [ + 1, + 1, + 4096 ] ], [ @@ -780,230 +738,164 @@ 1, 64 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_20", [ 1, - 12294, + 4096, 1024 ] - ] - ] - } - }, - "sha256": "b5828ac85b0f42118e79f2e83cbe05021d2b68f9a3c167f0ac014a3026799cc2" - }, - "den_joint_04": { - "file": "den_joint_04.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_21", [ 1, - 12294, + 1, 1024 ] ], [ - "args_1", + "args_22", [ 1, 6144 ] ], [ - "args_2", + "args_23", [ 1, + 1024, 64 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_24", [ 1, - 12294, + 64, 1024 ] - ] - ] - } - }, - "sha256": "ea5c57b89c8d8bbc5144d5d487e20fc49daf25308bccf853bb3c9d4581ae33fe" - }, - "den_joint_05": { - "file": "den_joint_05.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_25", [ 1, - 12294, + 64, 1024 ] ], [ - "args_1", + "args_26", [ 1, - 6144 + 64, + 1024 ] ], [ - "args_2", + "args_27", [ 1, + 1024, 64 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_28", [ 1, - 12294, + 64, 1024 ] - ] - ] - } - }, - "sha256": "f339d8b79e5d4c2fc408c2d42f7ce959b81821cad37861b2a1294b8397e19997" - }, - "den_joint_06": { - "file": "den_joint_06.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_29", [ 1, - 12294, - 1024 + 1024, + 64 ] ], [ - "args_1", + "args_3", [ 1, - 6144 + 1, + 1024 ] ], [ - "args_2", + "args_30", [ 1, - 64 + 64, + 4096 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_31", [ 1, - 12294, - 1024 + 4096, + 64 ] - ] - ] - } - }, - "sha256": "30d97e62ac4fc28882903961c88e65e6fe540d61e95504651b44d7cb8f341c4c" - }, - "den_joint_07": { - "file": "den_joint_07.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_32", [ 1, - 12294, + 64, 1024 ] ], [ - "args_1", + "args_4", [ 1, - 6144 + 1, + 1024 ] ], [ - "args_2", + "args_5", [ 1, - 64 + 1024, + 128 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_6", [ 1, - 12294, - 1024 + 1024, + 128 ] - ] - ] - } - }, - "sha256": "0c7b243e55c69b6cdd9c5e7bbef313f6cc6be0c0f7ad32e428187e22b2183a67" - }, - "den_joint_08": { - "file": "den_joint_08.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_7", [ 1, - 12294, - 1024 + 128, + 512 ] ], [ - "args_1", + "args_8", [ 1, - 6144 + 1024, + 1024 ] ], [ - "args_2", + "args_9", [ 1, - 64 + 1, + 1024 ] ] ], @@ -1019,13 +911,39 @@ ] } }, - "sha256": "0eb17bea8ae04ea0d88d79b327515e1a106bb5339d72440a8ec63712865e3f27" + "shared_blocks": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ], + "block_weights": "den_joint_w.f32", + "first_weight_input": 3, + "sha256": "f4c96ff1c3d9ca92de152978107c5280d4212591608cd19c6bbab8c301a291be" }, - "den_joint_09": { - "file": "den_joint_09.tflite", - "bytes": 56134100, + "den_joint_23": { + "file": "den_joint_23.tflite", + "bytes": 56235684, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -1046,6 +964,13 @@ ], [ "args_2", + [ + 1, + 1024 + ] + ], + [ + "args_3", [ 1, 64 @@ -1057,43 +982,60 @@ "output_0", [ 1, - 12294, - 1024 + 8192, + 16 ] - ] - ] - } - }, - "sha256": "353ece07e6b4888d80ab08e2bdd6dc2dd456f213ca87694f6d77f987d5d4debc" + ], + [ + "output_1", + [ + 1, + 1, + 5 + ] + ] + ] + } + }, + "sha256": "53420d4ff17eeee40d89480fecc549d578b8877a386821eb76c2754a4737ebdb" }, - "den_joint_10": { - "file": "den_joint_10.tflite", - "bytes": 56134100, + "oct": { + "file": "oct.tflite", + "bytes": 204247600, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { - "serving_default": { + "L64": { "inputs": [ [ "args_0", [ 1, - 12294, - 1024 + 64, + 3 ] ], [ "args_1", [ 1, - 6144 + 64, + 1024 ] ], [ "args_2", [ 1, - 64 + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 16 ] ] ], @@ -1102,43 +1044,43 @@ "output_0", [ 1, - 12294, - 1024 + 64, + 8 ] ] ] - } - }, - "sha256": "0c7e15340ea4212369ae85f3f9db625f21ccd1747038e0725675c5a7ca32ac07" - }, - "den_joint_11": { - "file": "den_joint_11.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { + }, + "L256": { "inputs": [ [ "args_0", [ 1, - 12294, - 1024 + 256, + 3 ] ], [ "args_1", [ 1, - 6144 + 256, + 1024 ] ], [ "args_2", [ 1, - 64 + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 16 ] ] ], @@ -1147,43 +1089,43 @@ "output_0", [ 1, - 12294, - 1024 + 256, + 8 ] ] ] - } - }, - "sha256": "76cc264ae1ce820749f9dd4758e07fa548ee805a7487cbd4ba342eccc523d779" - }, - "den_joint_12": { - "file": "den_joint_12.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { + }, + "L1024": { "inputs": [ [ "args_0", [ 1, - 12294, - 1024 + 1024, + 3 ] ], [ "args_1", [ 1, - 6144 + 1024, + 1024 ] ], [ "args_2", [ 1, - 64 + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 16 ] ] ], @@ -1192,43 +1134,43 @@ "output_0", [ 1, - 12294, - 1024 + 1024, + 8 ] ] ] - } - }, - "sha256": "75ff657c9c7cca82e45ec36fc259cde4cfc90829f09639cdc749d8bddce73f27" - }, - "den_joint_13": { - "file": "den_joint_13.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { + }, + "L4096": { "inputs": [ [ "args_0", [ 1, - 12294, - 1024 + 4096, + 3 ] ], [ "args_1", [ 1, - 6144 + 4096, + 1024 ] ], [ "args_2", [ 1, - 64 + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 16 ] ] ], @@ -1237,43 +1179,43 @@ "output_0", [ 1, - 12294, - 1024 + 4096, + 8 ] ] ] - } - }, - "sha256": "9ecae224a9b4962e1e1b1d719942ea89182c77b1f1cf629017f7117d9dded5b2" - }, - "den_joint_14": { - "file": "den_joint_14.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { + }, + "L8192": { "inputs": [ [ "args_0", [ 1, - 12294, - 1024 + 8192, + 3 ] ], [ "args_1", [ 1, - 6144 + 8192, + 1024 ] ], [ "args_2", [ 1, - 64 + 256 + ] + ], + [ + "args_3", + [ + 1, + 8192, + 16 ] ] ], @@ -1282,20 +1224,20 @@ "output_0", [ 1, - 12294, - 1024 + 8192, + 8 ] ] ] } }, - "sha256": "59f957bdcbcef54f90ac90fc2ed9ff52f40fa93afeb589f6fa377c734ddb4c5c" + "sha256": "6dab2c2dc79013e798ffabf41dea246bf447cee7bc9336430e8d773162a7cda7" }, - "den_joint_15": { - "file": "den_joint_15.tflite", - "bytes": 56134100, + "gs_0": { + "file": "gs_0.tflite", + "bytes": 122196424, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -1303,22 +1245,24 @@ "args_0", [ 1, - 12294, - 1024 + 2048, + 3 ] ], [ "args_1", [ 1, - 6144 + 2048, + 1024 ] ], [ "args_2", [ 1, - 64 + 8192, + 16 ] ] ], @@ -1327,20 +1271,20 @@ "output_0", [ 1, - 12294, + 2048, 1024 ] ] ] } }, - "sha256": "4839713f18df82b2dd53ed2d605a9d98240bee6bebc00e281a1316246b5ffaaa" + "sha256": "a0b0471f4d544c24708298a936d14fe40a4983d0b299dd4826605d32942bc8ec" }, - "den_joint_16": { - "file": "den_joint_16.tflite", - "bytes": 56134100, + "gs_mid": { + "file": "gs_mid.tflite", + "bytes": 92380, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -1348,7 +1292,7 @@ "args_0", [ 1, - 12294, + 2048, 1024 ] ], @@ -1356,327 +1300,176 @@ "args_1", [ 1, - 6144 + 8192, + 16 ] ], [ - "args_2", + "args_10", [ 1, - 64 + 1024, + 1024 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_11", [ 1, - 12294, + 1, 1024 ] - ] - ] - } - }, - "sha256": "a8cc4d07fe9830a8f2fc72420db955c6d08f6093fc82ea9b329df4d3275599e2" - }, - "den_joint_17": { - "file": "den_joint_17.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_12", [ 1, - 12294, + 1024, 1024 ] ], [ - "args_1", + "args_13", [ 1, - 6144 + 1, + 1024 ] ], [ - "args_2", + "args_14", [ 1, - 64 + 16, + 1024 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_15", [ 1, - 12294, + 1, 1024 ] - ] - ] - } - }, - "sha256": "29c8966f6b967250b818ebc902b0b1c6070436700c4cd52ad65ab26a3b102387" - }, - "den_joint_18": { - "file": "den_joint_18.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_16", [ 1, - 12294, + 16, 1024 ] ], [ - "args_1", + "args_17", [ 1, - 6144 + 1, + 1024 ] ], [ - "args_2", + "args_18", [ 1, + 1, + 16, 64 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_19", [ 1, - 12294, - 1024 + 1, + 16, + 64 ] - ] - ] - } - }, - "sha256": "e5f035dbfa13fcbb92f0212d4c773b8595ae1765b6658a9ecac11dfc80059b21" - }, - "den_joint_19": { - "file": "den_joint_19.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_2", [ 1, - 12294, + 1024, 1024 ] ], [ - "args_1", + "args_20", [ 1, - 6144 + 1024, + 1024 ] ], [ - "args_2", + "args_21", [ 1, - 64 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 12294, 1024 ] - ] - ] - } - }, - "sha256": "48394c1fbb680f7bb68ccf3e0253afc170ef47f0f14dc1163d0c84fb9c46222d" - }, - "den_joint_20": { - "file": "den_joint_20.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ - [ - "args_0", - [ - 1, - 12294, - 1024 - ] - ], - [ - "args_1", - [ - 1, - 6144 - ] ], [ - "args_2", + "args_22", [ 1, - 64 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 12294, 1024 ] - ] - ] - } - }, - "sha256": "2be313d01a72354c1b5dbc291ff1dcecf6345b02e7805e9214c019094762d063" - }, - "den_joint_21": { - "file": "den_joint_21.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_23", [ 1, - 12294, + 1, 1024 ] ], [ - "args_1", + "args_24", [ 1, - 6144 + 1024, + 4096 ] ], [ - "args_2", - [ - 1, - 64 - ] - ] - ], - "outputs": [ - [ - "output_0", + "args_25", [ 1, - 12294, - 1024 - ] - ] - ] - } - }, - "sha256": "c9c05edb15046307eb06de930d335bdc3b532f5eb9a79314664bb6ecfb087e4e" - }, - "den_joint_22": { - "file": "den_joint_22.tflite", - "bytes": 56134100, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ - [ - "args_0", - [ 1, - 12294, - 1024 + 4096 ] ], [ - "args_1", + "args_26", [ 1, - 6144 + 4096, + 1024 ] ], [ - "args_2", + "args_27", [ 1, - 64 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 12294, 1024 ] - ] - ] - } - }, - "sha256": "e7674dcda5bef2fca961742e0cb58736e9bed70485a2cad96080bbcddea3490b" - }, - "den_joint_23": { - "file": "den_joint_23.tflite", - "bytes": 56235684, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_28", [ 1, - 12294, + 1024, 1024 ] ], [ - "args_1", + "args_29", [ 1, - 6144 - ] - ], - [ - "args_2", - [ 1, 1024 ] @@ -1685,140 +1478,62 @@ "args_3", [ 1, - 64 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 8192, - 16 + 1024 ] ], [ - "output_1", + "args_30", [ 1, - 1, - 5 - ] - ] - ] - } - }, - "sha256": "53420d4ff17eeee40d89480fecc549d578b8877a386821eb76c2754a4737ebdb" - }, - "oct": { - "file": "oct.tflite", - "bytes": 204247600, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "L64": { - "inputs": [ - [ - "args_0", - [ - 1, - 64, - 3 + 1024, + 1024 ] ], [ - "args_1", + "args_31", [ 1, - 64, + 1, 1024 ] ], [ - "args_2", + "args_32", [ 1, - 256 + 1024, + 1024 ] ], [ - "args_3", + "args_33", [ 1, - 8192, - 16 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ - 1, - 64, - 8 - ] - ] - ] - }, - "L256": { - "inputs": [ - [ - "args_0", - [ 1, - 256, - 3 - ] - ], - [ - "args_1", - [ - 1, - 256, 1024 ] ], [ - "args_2", + "args_34", [ 1, - 256 + 1, + 16, + 64 ] ], [ - "args_3", + "args_35", [ 1, - 8192, - 16 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 256, - 8 - ] - ] - ] - }, - "L1024": { - "inputs": [ - [ - "args_0", - [ - 1, - 1024, - 3 + 16, + 64 ] ], [ - "args_1", + "args_36", [ 1, 1024, @@ -1826,389 +1541,191 @@ ] ], [ - "args_2", + "args_37", [ 1, - 256 - ] - ], - [ - "args_3", - [ 1, - 8192, - 16 + 1024 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_38", [ 1, 1024, - 8 - ] - ] - ] - }, - "L4096": { - "inputs": [ - [ - "args_0", - [ - 1, - 4096, - 3 + 1024 ] ], [ - "args_1", + "args_39", [ 1, - 4096, + 1, 1024 ] ], [ - "args_2", + "args_4", [ 1, - 256 + 1024, + 1024 ] ], [ - "args_3", + "args_40", [ 1, - 8192, - 16 + 16, + 1024 ] - ] - ], - "outputs": [ + ], [ - "output_0", + "args_41", [ 1, - 4096, - 8 - ] - ] - ] - }, - "L8192": { - "inputs": [ - [ - "args_0", - [ 1, - 8192, - 3 + 1024 ] ], [ - "args_1", + "args_42", [ 1, - 8192, + 16, 1024 ] ], [ - "args_2", + "args_43", [ 1, - 256 + 1, + 1024 ] ], [ - "args_3", + "args_44", [ 1, - 8192, - 16 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 8192, - 8 + 16, + 64 ] - ] - ] - } - }, - "sha256": "6dab2c2dc79013e798ffabf41dea246bf447cee7bc9336430e8d773162a7cda7" - }, - "gs_0": { - "file": "gs_0.tflite", - "bytes": 122497768, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_45", [ 1, - 8192, - 3 + 1, + 16, + 64 ] ], [ - "args_1", + "args_46", [ 1, - 8192, + 1024, 1024 ] ], [ - "args_2", + "args_47", [ 1, - 8192, - 16 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ - 1, - 8192, - 1024 - ] - ] - ] - } - }, - "sha256": "244ae2ffc6d510c8eab99d2381cb20edb8a7c1715272f7a7efeba5fcd77cfa39" - }, - "gs_1": { - "file": "gs_1.tflite", - "bytes": 118281876, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ - [ - "args_0", - [ 1, - 8192, 1024 ] ], [ - "args_1", + "args_48", [ 1, - 8192, - 16 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 8192, 1024 ] - ] - ] - } - }, - "sha256": "08e405e5b545d61b02b9f053d79c09f1300b57cd0f4469ffa65853eb8178bd4d" - }, - "gs_2": { - "file": "gs_2.tflite", - "bytes": 118281876, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_49", [ 1, - 8192, + 1, 1024 ] ], [ - "args_1", + "args_5", [ 1, - 8192, - 16 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 8192, 1024 ] - ] - ] - } - }, - "sha256": "4280e07069d2a6cd271e54477164b8b632d588030e46909a1854a8b35e30eb93" - }, - "gs_3": { - "file": "gs_3.tflite", - "bytes": 118281876, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_50", [ 1, - 8192, - 1024 + 1024, + 4096 ] ], [ - "args_1", + "args_51", [ 1, - 8192, - 16 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 8192, - 1024 + 4096 ] - ] - ] - } - }, - "sha256": "5dcd7ecb298a160af606530df20f018b91384cd25e06fe25a0b42401cba7e651" - }, - "gs_4": { - "file": "gs_4.tflite", - "bytes": 118281876, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_52", [ 1, - 8192, + 4096, 1024 ] ], [ - "args_1", + "args_53", [ 1, - 8192, - 16 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 8192, 1024 ] - ] - ] - } - }, - "sha256": "37541cee18d0de7b62e104391af8f61eac192424f58a9613cd8281fe6c23f472" - }, - "gs_5": { - "file": "gs_5.tflite", - "bytes": 118281876, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_6", [ 1, - 8192, + 1024, 1024 ] ], [ - "args_1", + "args_7", [ 1, - 8192, - 16 - ] - ] - ], - "outputs": [ - [ - "output_0", - [ 1, - 8192, 1024 ] - ] - ] - } - }, - "sha256": "80972de6ec50f8d6f1ace44c47f076d0f214c0cdb99b6467d37e82af1de940c3" - }, - "gs_6": { - "file": "gs_6.tflite", - "bytes": 118281876, - "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", - "signatures": { - "serving_default": { - "inputs": [ + ], [ - "args_0", + "args_8", [ 1, - 8192, - 1024 + 1, + 16, + 64 ] ], [ - "args_1", + "args_9", [ 1, - 8192, - 16 + 1, + 16, + 64 ] ] ], @@ -2217,20 +1734,30 @@ "output_0", [ 1, - 8192, + 2048, 1024 ] ] ] } }, - "sha256": "50dcc38afa5253d24fb792798db27327905b766404506405ead8e525cb0ec177" + "shared_blocks": [ + 1, + 2, + 3, + 4, + 5, + 6 + ], + "block_weights": "gs_w.f32", + "first_weight_input": 2, + "sha256": "52e7ab4506909972f66ebd771571833a7740e79d17aa79bc59273eec9d846308" }, "gs_7": { "file": "gs_7.tflite", - "bytes": 120252780, + "bytes": 119951436, "accelerator": "gpu", - "precision": "fp16_with_fp32_accum", + "precision": "fp16", "signatures": { "serving_default": { "inputs": [ @@ -2238,7 +1765,7 @@ "args_0", [ 1, - 8192, + 2048, 1024 ] ], @@ -2256,14 +1783,14 @@ "output_0", [ 1, - 8192, + 2048, 480 ] ] ] } }, - "sha256": "ef75898f78654d4d10692b333dc1e12c860ee54138795aece8e2306ba76217b7" + "sha256": "9a30c4304de8c6723d3860c139e0f5bad75bf773fe931b3d60bae9a51521592a" } }, "side_files": { @@ -2301,13 +1828,27 @@ 3 ], "dtype": "float32_le" + }, + "den_joint_w.f32": { + "shape": [ + 22, + 13978624 + ], + "dtype": "float32_le" + }, + "gs_w.f32": { + "shape": [ + 6, + 29464576 + ], + "dtype": "float32_le" } }, "json_files": [ "vae_bn.json", "gs_rep.json" ], - "total_graph_bytes": 3837368884, + "total_graph_bytes": 1892333556, "repos": { "main": "litert-community/Idol", "dino": "litert-community/DINOv3-ViT-H16plus-Idol-LiteRT", diff --git a/python/export_graphs.py b/python/export_graphs.py index a5b18bf7aa2bea1715cd971f82bd3b1fb62865ea..f84bab7dbab77fcbc36c98872a454e3379e61295 100644 --- a/python/export_graphs.py +++ b/python/export_graphs.py @@ -179,7 +179,7 @@ def part_gs(out, ref, do_export, chunk=512): if do_export: res['export'] = {} for i, p in enumerate(parts): - args = (anchors, pe, latent) if i == 0 else (torch.randn(1, 8192, 1024), latent) + args = (anchors, pe, latent) if i == 0 else (torch.randn(1, anchors.shape[1], 1024), latent) res['export'][f'gs_{i}'] = export(p, args, os.path.join(out, f'gs_{i}.tflite')) return res diff --git a/python/export_shared_gauss.py b/python/export_shared_gauss.py new file mode 100644 index 0000000000000000000000000000000000000000..f9bc72c24a0c7a239a1b06b9e2dde6c275757742 --- /dev/null +++ b/python/export_shared_gauss.py @@ -0,0 +1,82 @@ +"""The state lever for the Gaussian decoder (round 2): one graph for its six middle graphs gs_1 .. gs_6 (two blocks +each, the same shapes), each graph's weights as runtime inputs, so the phone compiles it once per photo instead of six +times (in the app the Pixel 11 Pro compiled each gs graph in 8.2-8.7 s, the Galaxy S26 in 1.3 s). + +Writes /gs_mid.tflite (no weights; inputs h, latent, then the two blocks' weight inputs, each block in +idol_graphs.GAUSS_BLOCK_WEIGHT_NAMES order), /gs_w.f32 (little-endian float32, graphs 1..6 one after another, +each graph's weight inputs in input order, flattened) and /shared_gs.json, and checks the shared module against +every middle graph's own module in PyTorch on one random input. + + IDOL_SDK= python export_shared_gauss.py --out models/state --anchors 2048 +""" +import argparse +import json +import os +import sys +import time + +import numpy as np +import torch + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import export_graphs as EG # noqa: E402 (also puts the idol-sdk on the path) +import f0_export as F0 # noqa: E402 +import idol_graphs as IG # noqa: E402 + +LATENT_TOKENS, LATENT_CHANNELS = 8192, 16 +FIRST_WEIGHT_INPUT = 2 # gs_mid's inputs before its weights: h, latent + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--out', required=True) + ap.add_argument('--anchors', type=int, default=8192) + ap.add_argument('--chunk', type=int, default=512) + a = ap.parse_args() + os.makedirs(a.out, exist_ok=True) + torch.set_grad_enabled(False) + dgs = EG.load_decoder('decoder_gs') + graphs = list(range(1, len(EG.GS_SPLIT) - 1)) + mids = [IG.Gauss(dgs, *EG.GS_SPLIT[i], a.chunk).eval() for i in graphs] + shared = IG.GaussShared(mids[0]).eval() + shapes = [list(t.shape) for t in IG.gauss_shared_weights(mids[0])] + torch.manual_seed(0) + h = torch.randn(1, a.anchors, 1024) + latent = torch.randn(1, LATENT_TOKENS, LATENT_CHANNELS) + worst = 0.0 + wpath = os.path.join(a.out, 'gs_w.f32') + with open(wpath, 'wb') as f: + for i, g in zip(graphs, mids): + assert not g.first and not g.last and len(g.blocks) == shared.n, i + for b in g.blocks: + assert (b.e1, b.e2, b.e3) == shared.eps, i + assert (b.self_attn.h, b.self_attn.c, b.self_attn.d, b.self_attn.chunk) == shared.sa, i + assert (b.cross_attn.h, b.cross_attn.c, b.cross_attn.d, b.cross_attn.chunk) == shared.ca, i + w = IG.gauss_shared_weights(g) + assert [list(t.shape) for t in w] == shapes, i + got, want = shared(h, latent, *w), g(h, latent) + rel = float((got - want).norm() / want.norm()) + worst = max(worst, rel) + print(f'gs_{i}: shared vs own module rel L2 {rel:.2e}', flush=True) + for t in w: + f.write(t.numpy().astype('/den_joint_mid.tflite (no weights; inputs h, t_mod, gate, then the 30 weight inputs of +idol_graphs.SHARED_WEIGHT_NAMES) and /den_joint_w.f32 (little-endian float32, blocks 1..22 one after another, +each block's inputs in that order, flattened), and checks the shared module against every block's own module in +PyTorch on one random input. + + IDOL_SDK= python export_shared_joint.py --out models/state +""" +import argparse +import json +import os +import sys +import time + +import numpy as np +import torch +from safetensors.torch import load_file + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import f0_export as F0 # noqa: E402 +import idol_graphs as IG # noqa: E402 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--out', required=True) + ap.add_argument('--chunk', type=int, default=512) + a = ap.parse_args() + os.makedirs(a.out, exist_ok=True) + torch.set_grad_enabled(False) + den, _ = F0.load_denoiser() + den = den.float() + lora = load_file(F0.snap('speridlabs/idol-turbo', 'idol_turbo_lora.safetensors')) + blocks = list(range(1, len(den.blocks) - 1)) + mids = [IG.DenJoint(den, k, lora, a.chunk).eval() for k in blocks] + shared = IG.DenJointShared(mids[0]).eval() + ref = mids[0].block + shapes = [list(t.shape) for t in IG.shared_block_weights(mids[0])] + L = IG.DEN_TOKENS + IG.COND_TOKENS + 1 + torch.manual_seed(0) + h = torch.randn(1, L, 1024) + t_mod = torch.randn(1, 6144) * 0.1 + gate = IG.lora_gate('slow') + worst = 0.0 + wpath = os.path.join(a.out, 'den_joint_w.f32') + with open(wpath, 'wb') as f: + for k, jb in zip(blocks, mids): + b = jb.block + assert b.modulation and (b.c, b.h, b.d, b.eps1, b.eps2, b.repo.eps) == \ + (ref.c, ref.h, ref.d, ref.eps1, ref.eps2, ref.repo.eps), k + w = IG.shared_block_weights(jb) + assert [list(t.shape) for t in w] == shapes, k + got, want = shared(h, t_mod, gate, *w), jb(h, t_mod, gate) + rel = float((got - want).norm() / want.norm()) + worst = max(worst, rel) + print(f'block {k:2d}: shared vs own module rel L2 {rel:.2e}', flush=True) + for t in w: + f.write(t.numpy().astype(' h, the + block's weights as runtime inputs (shared_block_weights order). Same arithmetic as DenBlock.forward; compiled once + per photo on the phone instead of once per block.""" + + def __init__(self, ref_joint): + super().__init__() + b = ref_joint.block + self.c, self.h, self.d, self.chunk = b.c, b.h, b.d, b.chunk + self.eps1, self.eps2, self.repo_eps, self.repo_half = b.eps1, b.eps2, b.repo.eps, b.repo.half + + def forward(self, x, mod, gate, *w): + W = dict(zip(SHARED_WEIGHT_NAMES, w)) + c, H, d = self.c, self.h, self.d + bsz, L, _ = x.shape + mm = torch.matmul + g = gate[:, None, :] + hr = G.safe_layer_norm(x, self.repo_eps, W['repo_norm_w'], W['repo_norm_b']) + a = mm(F.silu(mm(hr, W['repo_gate_t'])) * mm(hr, W['repo_content_t']), W['repo_ang_t']) + a = a.reshape(bsz, L, H, self.repo_half) + cos, sin = torch.cos(a), torch.sin(a) + m = mod + W['shift'] + shift_msa, scale_msa, gate_msa = m[:, 0:c], m[:, c:2 * c], m[:, 2 * c:3 * c] + shift_mlp, scale_mlp, gate_mlp = m[:, 3 * c:4 * c], m[:, 4 * c:5 * c], m[:, 5 * c:6 * c] + hh = G.safe_layer_norm(x, self.eps1) * (1 + scale_msa.unsqueeze(1)) + shift_msa.unsqueeze(1) + half = d // 2 + + def rope(t): + t0, t1 = t[..., :half], t[..., half:] + return torch.cat([t0 * cos - t1 * sin, t0 * sin + t1 * cos], dim=-1) + + ax = mm(hh, W['lqkv_a_t']) * g + q = G.safe_rms_unit(rope((mm(hh, W['q_t']) + W['q_b'] + mm(ax, W['lq_b_t'])).reshape(bsz, L, H, d))) * W['gq'] + k = G.safe_rms_unit(rope((mm(hh, W['k_t']) + W['k_b'] + mm(ax, W['lk_b_t'])).reshape(bsz, L, H, d))) * W['gk'] + v = (mm(hh, W['v_t']) + W['v_b'] + mm(ax, W['lv_b_t'])).reshape(bsz, L, H, d) + o = G.attention(q.permute(0, 2, 1, 3), k.permute(0, 2, 1, 3), v.permute(0, 2, 1, 3), self.chunk) + o = o.reshape(bsz, L, c) + x = x + (mm(o, W['out_t']) + W['out_b'] + mm(mm(o, W['lout_a_t']) * g, W['lout_b_t'])) * gate_msa.unsqueeze(1) + hh = G.safe_layer_norm(x, self.eps2) * (1 + scale_mlp.unsqueeze(1)) + shift_mlp.unsqueeze(1) + u = G._gelu_tanh(mm(hh, W['fc1_t']) + W['fc1_b'] + mm(mm(hh, W['lfc1_a_t']) * g, W['lfc1_b_t'])) + return x + (mm(u, W['fc2_t']) + W['fc2_b'] + mm(mm(u, W['lfc2_a_t']) * g, W['lfc2_b_t'])) * gate_mlp.unsqueeze(1) + + +# ---------------------------------------------------------------- shared middle Gaussian decoder graph (state lever) + +# One GaussianBlockGPU's weights as graph inputs, in input order: self-attention, cross-attention, norm2's affine, MLP. +# Linear weights are passed transposed ([1, in, out]), as for the shared joint block. +GAUSS_BLOCK_WEIGHT_NAMES = ['sa_q_t', 'sa_q_b', 'sa_k_t', 'sa_k_b', 'sa_v_t', 'sa_v_b', 'sa_gq', 'sa_gk', 'sa_out_t', + 'sa_out_b', 'ca_q_t', 'ca_q_b', 'ca_k_t', 'ca_k_b', 'ca_v_t', 'ca_v_b', 'ca_gq', 'ca_gk', + 'ca_out_t', 'ca_out_b', 'n2_w', 'n2_b', 'fc1_t', 'fc1_b', 'fc2_t', 'fc2_b'] + + +def gauss_block_weights(gb): + """One GaussianBlockGPU's weights in GAUSS_BLOCK_WEIGHT_NAMES order (float32 tensors, the graph input shapes).""" + def tw(lin): + return lin.weight.detach().float().t().contiguous()[None] + + def row(v): + return v.detach().float().reshape(1, 1, -1) + + def mha(m): + return [tw(m.q), row(m.q.bias), tw(m.k), row(m.k.bias), tw(m.v), row(m.v.bias), m.gq.detach().float(), + m.gk.detach().float(), tw(m.out), row(m.out.bias)] + + assert gb.n2w is not None and gb.n2b is not None + return (mha(gb.self_attn) + mha(gb.cross_attn) + + [row(gb.n2w), row(gb.n2b), tw(gb.fc1), row(gb.fc1.bias), tw(gb.fc2), row(gb.fc2.bias)]) + + +def gauss_shared_weights(g): + """A middle Gauss graph's weights: its blocks' gauss_block_weights one after another.""" + return [t for b in g.blocks for t in gauss_block_weights(b)] + + +class GaussShared(nn.Module): + """One graph for the Gaussian decoder's middle graphs (gs_1 .. gs_6, two blocks each): (h [1, A, 1024], + latent [1, 8192, 16], *weights) -> h, the blocks' weights as runtime inputs (gauss_shared_weights order). Same + arithmetic as Gauss.forward for a middle graph; compiled once per photo on the phone instead of six times.""" + + def __init__(self, ref): + super().__init__() + assert not ref.first and not ref.last + b0 = ref.blocks[0] + self.n = len(ref.blocks) + self.eps = (b0.e1, b0.e2, b0.e3) + self.sa = (b0.self_attn.h, b0.self_attn.c, b0.self_attn.d, b0.self_attn.chunk) + self.ca = (b0.cross_attn.h, b0.cross_attn.c, b0.cross_attn.d, b0.cross_attn.chunk) + + @staticmethod + def _mha(x, src, W, p, heads, c, d, chunk): + mm = torch.matmul + b, lq, _ = x.shape + lk = src.shape[1] + q = G.safe_rms_unit((mm(x, W[p + 'q_t']) + W[p + 'q_b']).reshape(b, lq, heads, d)) * W[p + 'gq'] + k = G.safe_rms_unit((mm(src, W[p + 'k_t']) + W[p + 'k_b']).reshape(b, lk, heads, d)) * W[p + 'gk'] + v = (mm(src, W[p + 'v_t']) + W[p + 'v_b']).reshape(b, lk, heads, d) + o = G.attention(q.permute(0, 2, 1, 3), k.permute(0, 2, 1, 3), v.permute(0, 2, 1, 3), chunk) + return mm(o.reshape(b, lq, c), W[p + 'out_t']) + W[p + 'out_b'] + + def forward(self, h, latent, *w): + n = len(GAUSS_BLOCK_WEIGHT_NAMES) + assert len(w) == n * self.n, len(w) + e1, e2, e3 = self.eps + for i in range(self.n): + W = dict(zip(GAUSS_BLOCK_WEIGHT_NAMES, w[i * n:(i + 1) * n])) + x = G.safe_layer_norm(h, e1) + h = h + self._mha(x, x, W, 'sa_', *self.sa) + h = h + self._mha(G.safe_layer_norm(h, e2, W['n2_w'], W['n2_b']), latent, W, 'ca_', *self.ca) + u = G._gelu_tanh(torch.matmul(G.safe_layer_norm(h, e3), W['fc1_t']) + W['fc1_b']) + h = h + (torch.matmul(u, W['fc2_t']) + W['fc2_b']) + return h diff --git a/python/idol_runner.py b/python/idol_runner.py index 9e80c492712020c12fb39ff4360f5ed0c04f9e7a..a9c07bd3022b944053072f48d0c3aec804a88d4c 100644 --- a/python/idol_runner.py +++ b/python/idol_runner.py @@ -50,9 +50,46 @@ class IdolLiteRT: self.times = {} # graphs that run in fp32 precision (their fp16 form fails a CONTRACT.md line): the manifest's, as the app does self.fp32_graphs = set() + # anchor points of the octree / Gaussian decoder: the first input of gs_0 is [1, anchors, 3] (8,192 by default) + self.num_points = 8192 + # shared graphs (den_joint_mid, gs_mid: the middle blocks' weights as runtime inputs) and their weights files + self.shared = {} if os.path.exists(f('manifest.json')): man = json.load(open(f('manifest.json'))) self.fp32_graphs = {n for n, g in man['graphs'].items() if g['precision'] == 'fp32'} + if 'gs_0' in man['graphs']: + self.num_points = int(next(iter(man['graphs']['gs_0']['signatures'].values()))['inputs'][0][1][1]) + for n, g in man['graphs'].items(): + if g.get('shared_blocks'): + ins = next(iter(g['signatures'].values()))['inputs'] + first = g.get('first_weight_input', 3) # the inputs before the weights + shapes = [s for _, s in sorted(ins, key=lambda t: int(t[0].split('_')[-1]))][first:] + w = np.memmap(f(g['block_weights']), '= 0: # by name: the signature lists 33 inputs as args_0, args_1, args_10, ... + args = [h, t_mod, gate] + self.block_weights(name, block) + h = self.run1(name, {f'args_{i}': a for i, a in enumerate(args)})[0] + else: + h = self.run1(name, [h, t_mod, gate])[0] v, vc = self.run1('den_joint_23', [h, t_mod, t_emb, gate]) self._t('denoiser', t0) return v, vc @@ -155,12 +196,13 @@ class IdolLiteRT: def anchors(self, latent, noise, rec=None): t0 = time.time() - pool, leaf = torch.from_numpy(noise['octree_pool']), torch.from_numpy(noise['leaf_pool']) + pool = torch.from_numpy(noise['octree_pool']) + leaf = torch.from_numpy(noise['leaf_pool'][:, :self.num_points]) def probs_fn(cells, level): logits = torch.from_numpy(self.octree_logits(cells.numpy(), level, latent)) return torch.softmax(logits.float(), dim=-1) - out = GL.octree_descend(probs_fn, pool, leaf, rec=rec).numpy() + out = GL.octree_descend(probs_fn, pool, leaf, num_points=self.num_points, rec=rec).numpy() self._t('octree', t0) return out @@ -168,8 +210,12 @@ class IdolLiteRT: t0 = time.time() pe = IG.point_pe(torch.from_numpy(anchors[0])).numpy()[None] h = self.run1('gs_0', [anchors, pe, latent])[0] - for i in range(1, 8): - h = self.run1(f'gs_{i}', [h, latent])[0] + for name, block in self.gs_steps: + if block >= 0: # by name: the signature lists the inputs as args_0, args_1, args_10, ... + args = [h, latent] + self.block_weights(name, block) + h = self.run1(name, {f'args_{i}': a for i, a in enumerate(args)})[0] + else: + h = self.run1(name, [h, latent])[0] self._t('gaussian_decoder', t0) g = GL.to_gaussians(torch.from_numpy(anchors), torch.from_numpy(h), self.rep, self.perturbation, torch.tensor(self.base_offset_scale)) diff --git a/python/make_manifest.py b/python/make_manifest.py index ae612ce7378ccfb30be08233bdb515a6d22b90a5..dc5f2b03e34054dbe6ed86d774e5c58ffa6dda81 100644 --- a/python/make_manifest.py +++ b/python/make_manifest.py @@ -3,6 +3,13 @@ accelerator and precision it runs at, which matting graph the app uses, and the Also writes gs_perturbation.f32 next to gs_perturbation.npy. python make_manifest.py models/v1 [--dis PATH_TO_dis.tflite] + +With den_joint_mid.tflite and den_joint_w.f32 in the folder (export_shared_joint.py) and no den_joint_01..22, the +manifest lists den_joint_mid in their place with `shared_blocks` [1..22], `block_weights` and `first_weight_input` +(the inputs before the weights), and the weights file as a side file [22, floats per block]. gs_mid.tflite and +gs_w.f32 (export_shared_gauss.py) with no gs_1..6 likewise stand for gs_1..6. + +--precision fp16 runs the GPU graphs at FP16 without fp32 accumulation (the default keeps FP16_WITH_FP32_ACCUM). """ import argparse import hashlib @@ -17,8 +24,15 @@ from lrt import sig_io # noqa: E402 FP32_GRAPHS = set() # graphs whose fp16 form fails its CONTRACT.md line (none since the key-chunked attention) CPU_GRAPHS = {'bir'} # BiRefNet: its deformable convolutions lower to GATHER_ND, which the LiteRT GPU does not run +# A CPU graph's working memory comes with its call, after the compile: how far MemAvailable fell from just before the +# compile to the low point of one call on the Galaxy S26 (LiteRT 2.3.0 CPU; device/g2/bir3.samples.txt: 6,492,924 kB +# before, 3,760,320 kB lowest = 2,732,604 kB). The app's memory check counts it before BiRefNet runs. +RUN_PEAK_BYTES = {'bir': 2_732_604 * 1024} ORDER = (['bir', 'dis'] + [f'dino_{i}' for i in range(4)] + ['vae_0', 'vae_1', 'vae_2', 'vae_3', 'vae_attn', 'vae_4'] + ['den_ctx', 'den_in'] + [f'den_joint_{k:02d}' for k in range(24)] + ['oct'] + [f'gs_{i}' for i in range(8)]) +# shared graph -> (the json its export script writes, its weights file, the graph name of block k) +SHARED = {'den_joint_mid': ('shared_joint.json', 'den_joint_w.f32', lambda k: f'den_joint_{k:02d}'), + 'gs_mid': ('shared_gs.json', 'gs_w.f32', lambda k: f'gs_{k}')} SIDE = {'den_pos_emb.f32': [1, 8192, 1024], 'dino_rope_cos.f32': [1, 1, 4101, 64], 'dino_rope_sin.f32': [1, 1, 4101, 64], 'gs_perturbation.f32': None} @@ -36,21 +50,42 @@ def main(): ap.add_argument('models') ap.add_argument('--dis', default=None) ap.add_argument('--no-sha', action='store_true') + ap.add_argument('--precision', choices=['fp16_with_fp32_accum', 'fp16'], default='fp16_with_fp32_accum', + help='precision of the GPU graphs') a = ap.parse_args() d = a.models pert = np.load(os.path.join(d, 'gs_perturbation.npy')).astype(' a 3D Gaussian splat (.ply, 262,144 Gaussians), with LiteRT CompiledModel. +"""Idol on LiteRT: one image -> a 3D Gaussian splat (.ply), with LiteRT CompiledModel. pip install -r requirements.txt - python run_idol.py photo.jpg --out out # downloads both repos with huggingface_hub + python run_idol.py photo.jpg --out out # downloads both repos with huggingface_hub (the default form) python run_idol.py photo.jpg --out out --models --dino +litert-community/Idol holds two forms: the default in the repo's root (65,536 Gaussians, the faster one) and the +first release's in `262k/` (262,144 Gaussians, the author's default size, closer to the author's output). Without +--models the runner downloads the default form only. + The DINOv3 encoder repo is gated: accept the DINOv3 License on its Hub page and log in (`hf auth login`) first. Background removal (the author's BiRefNet) runs on the CPU, everything else on the GPU (Metal on a Mac). --mode: `real` for photos (crop margin 1.7), `synthetic` for renders on a plain background (1.2). @@ -34,7 +38,7 @@ def main(): if models is None or dino is None: from huggingface_hub import snapshot_download if models is None: - models = snapshot_download(MAIN_REPO) + models = snapshot_download(MAIN_REPO, ignore_patterns=['262k/*']) if dino is None: dino_repo = json.load(open(os.path.join(models, 'manifest.json')))['repos']['dino'] dino = snapshot_download(dino_repo)