diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..f82bda2156eb5f0abc47184909352c8aec316642 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.venv/ +__pycache__/ +.pytest_cache/ +*.egg-info/ +data/ +runs/ +vendor/ +.cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000000000000000000000000000000000..74904b9325fe92bb0c1bff11932a5f80cb668cc1 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +## v0.1.0 — 2026-11-02 + +First public release of the local image-to-Gaussian student prototype. + +- Small FP32 student, cached-target training, held-out validation and standard Gaussian PLY IO. +- Experimental staged Idol Mac adapter; optional aligned InstantMesh/Hi3D mesh supervision. +- Windows/Linux CPU recipes and macOS CPU/Metal recipes. +- Apache-2.0 license and explicit third-party attribution. +- Synthetic-only demonstration checkpoint, procedural fixtures and verification evidence. + +No real teacher-trained model, 1-bit inference, full pretrained Idol validation or Mac execution of InstantMesh/Hi3D is claimed. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..9c3ec8815b53325f9bd9e0f22174fd1af3c80f9c --- /dev/null +++ b/NOTICE @@ -0,0 +1,11 @@ +Image3D Student v0.1.0 +Copyright 2026 Lucasgitguy-co. + +Original project source, documentation and procedural demonstration artifacts are released under Apache License 2.0. + +The optional Idol integration and Gaussian attribute conventions were developed with reference to +speridlabs/idol-sdk, revision 4702a7f00850e34b6e83a2170cc123c3e9570744 (Apache-2.0). +Its architecture is imported as an optional dependency; pretrained teacher weights are not redistributed. +Adaptations in this project: staged loading, query-chunk attention, explicit real-valued RoPE selection, +independent student training, coordinate-preserving Gaussian IO, and diagnostic rendering. +Upstream attribution is retained in licenses/Idol-APACHE-2.0.txt and THIRD_PARTY.md. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4c0792964a876e1604c83474c1eed7d95759690a --- /dev/null +++ b/README.md @@ -0,0 +1,198 @@ +--- +license: apache-2.0 +language: en +pipeline_tag: image-to-3d +tags: +- pytorch +- gaussian-splatting +- distillation +- research-prototype +- synthetic-data +model_name: Image3D Student v0.1.0 +--- +# Image3D Student v0.1.0 — first public prototype + +Train a compact image-to-Gaussian-splat student on cached teacher outputs, using Apple Metal or CPU. + +**First release: v0.1.0 · Apache-2.0.** See [Windows, Linux and macOS instructions](docs/USAGE.md), +[all publisher model guides](docs/ALL_MODELS.md), [license](LICENSE), [third-party scope](THIRD_PARTY.md), +and [release notes](CHANGELOG.md). + +**This is working research code, not a pretrained general-purpose image-to-3D model.** +The included demonstration checkpoint learned only procedural ellipsoids. It has not learned from Idol, +InstantMesh or Hi3D and should not be used as evidence of real-image quality. + +## What is implemented + +| Component | Status | +|---|---| +| Compact image encoder + coordinate decoder | Implemented, CPU and Metal training tested | +| Training, object-level validation, metrics and checkpoints | Implemented | +| Import/export of degree-zero Gaussian PLY | Implemented; preserves position, anisotropic scale, rotation, colour and opacity | +| Idol supervision | Import its PLY and the exact prepared input image | +| InstantMesh supervision | Import an aligned coarse triangle mesh as an auxiliary surface-occupancy target | +| Hi3D supervision | Import an aligned refined triangle mesh as a second auxiliary surface-occupancy target | +| Running Idol on Mac | Experimental staged adapter; tiny architecture components tested separately; full pretrained pipeline unverified | +| Running InstantMesh or Hi3D on Mac | **Not implemented**; official GPU implementations need porting | +| Binary/1-bit student | **Not implemented**; this small baseline uses FP32 | +| Differentiable rendering / multiview photometric loss | Not implemented in this first baseline | + +The three-teacher training interface is present, but a fully local way to **generate all three teachers' data** +is still unresolved. Importing an existing output does not mean the corresponding teacher can run on this Mac. + +## Start + +On the Mac where this project was built, the dependencies are already installed in the working directory. +From this delivered folder, `bash run-local.sh doctor` reuses that environment. Pass any command described below +to `bash run-local.sh` instead of `image3d`. On a different machine, use the setup steps below. + +Use Python 3.11–3.13. From this project directory: + +```sh +bash setup.sh +source .venv/bin/activate +image3d doctor +``` + +`setup.sh` installs the student dependencies only. It does not download teacher weights. +`doctor` checks this process's Metal access. Sandboxed terminals can report Metal unavailable even when a normal +macOS Terminal can use it. Choose `--device mps` to require GPU execution; `auto` may fall back to CPU. + +## Run the synthetic pipeline test + +```sh +image3d demo-data --out data/demo --count 12 --grid 8 +image3d train --manifest data/demo/manifest.jsonl --out runs/demo \ + --device mps --width 64 --steps 100 --lr 0.001 --eval-every 25 --allow-synthetic +image3d generate --checkpoint runs/demo/best.pt \ + --image data/demo/images/fixture-007.png --out runs/demo/student.ply --device mps +image3d preview --ply runs/demo/student.ply --out runs/demo/turntable.png +python -m pytest -q +``` + +The data split is seeded and groups all samples with the same object ID together. The commands refuse to overwrite +an existing run or exported PLY. Use a new directory for a new experiment. Ctrl-C saves completed optimizer steps +to `last.pt`. This version does not resume optimizer training from a checkpoint. + +With validation enabled, `best.pt` selects the lowest held-out distillation loss. `last.pt` always contains the final +completed step. With `--validation-fraction 0`, there is no `best.pt` and no generalization measurement. + +## Train with real teacher data + +1. Supply an image of one centered object and obtain its Idol PLY. +2. Cache the **exact input framing seen by Idol**, ideally the `*_input.png` saved by Idol. +3. Optionally attach aligned coarse/refined meshes for the same object and image. +4. Repeat for a dataset of different objects. Keep related views in the same `--object-id`. +5. Train and judge unseen objects, not only training loss. + +```sh +image3d cache --image teacher/item01/input.png --idol-ply teacher/item01/teacher.ply \ + --id item01-view01 --object-id item01 --out data/teachers --grid 16 + +image3d train --manifest data/teachers/manifest.jsonl --out runs/teachers-v1 \ + --device mps --steps 1000 --batch-size 1 + +image3d generate --checkpoint runs/teachers-v1/best.pt \ + --image your-centered-cutout.png --out results/object.ply --device mps +``` + +The student does not perform automatic background removal. Use already-framed cutouts, with the object scale +and orientation matching the training data. RGBA is composited on black, matching Idol's prepared images. +An arbitrary photo with clutter or different framing is outside the baseline's input contract. + +### Attach all three teachers + +```sh +image3d cache --image teacher/item02/input.png --idol-ply teacher/item02/teacher.ply \ + --coarse-mesh teacher/item02/instantmesh.obj --coarse-transform transforms/coarse.json \ + --fine-mesh teacher/item02/hi3d.glb --fine-transform transforms/fine.json \ + --id item02-view01 --object-id item02 --out data/teachers --grid 16 +``` + +Each transform file is a JSON 4×4 affine matrix mapping mesh coordinates into the **exported Idol PLY coordinate +frame**. Idol changes its internal z-up frame when exporting PLY; do not apply that rotation twice. +Only use an identity transform if the assets are already aligned. Example identity: + +```json +[[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]] +``` + +Targets must fit inside `[-0.5,0.5]^3`. Out-of-bounds targets are rejected, not silently normalized. +Check rotation, scale, translation and camera framing visually before combining teachers. Different teachers can +hallucinate different unseen geometry. This baseline does **not** automatically align shapes or resolve their disagreements. + +Geometry losses supervise auxiliary occupancy heads sharing the student features; they do not guarantee that final +Gaussian surfaces exactly follow the refined mesh. Joint rendering-based supervision and confidence filtering are future work. + +The manifest stores source labels, transforms, input/target hashes, object IDs and compression counts. A missing teacher +is masked out of the loss. Do not label fabricated outputs as real teacher supervision. + +## Experimental local Idol runner + +The adapter uses the official Idol architecture at pinned SDK commit +`4702a7f00850e34b6e83a2170cc123c3e9570744`. It loads background removal, conditioning, +denoising, octree decoding and Gaussian decoding in stages, deleting models between stages. +It replaces complex rotary embeddings with the upstream real-valued equivalent and chunks attention along the query +dimension while retaining all keys/values. CUDA compilation and batched classifier-free guidance are disabled. + +This reduces simultaneous working memory but **does not establish that full Idol fits in 16 GB**. +The encoders, their operators, loading peaks and full-size attention still need real-weight testing. +The runner has no silent CPU operator fallback, so unsupported operations surface as errors. + +Install code dependencies separately: + +```sh +uv pip install --python .venv/bin/python -e '.[teacher]' +``` + +Idol requires access to these gated encoders: + +- [DINOv3 ViT-H+](https://huggingface.co/facebook/dinov3-vith16plus-pretrain-lvd1689m) +- [FLUX.2-dev VAE](https://huggingface.co/black-forest-labs/FLUX.2-dev) + +Accept their terms using your own Hugging Face account. Set the local cache consistently before logging in: + +```sh +export HF_HOME="$PWD/.cache/huggingface" +hf auth login + +# Explicitly allows model downloads. Start with a transparent single-object PNG. +image3d teacher-idol --image your-cutout.png --out teacher/item01 \ + --cache "$HF_HOME" --allow-download --device mps --steps 20 --gaussians 32768 +``` + +The download command requires 16 GiB of free disk space as a conservative safety margin, not a measured model minimum. +At build time this Mac had approximately 7.4 GB available before dependency installation, so no model weights were downloaded. +With a complete existing cache, omit `--allow-download`: the runner then uses local files only. +20 steps is a reduced-cost experiment; it is not the teacher's default 50-step quality setting. +Use `--steps 50` once execution and memory have been verified. + +## Architecture and losses + +The baseline is deliberately small: a four-stage convolutional image encoder produces a shared feature vector. +A coordinate MLP predicts one Gaussian per cell in a normalized object cube, plus coarse/fine geometry logits. +The default 16³ grid has at most 4,096 Gaussians; grid 32 has at most 32,768. Grid 8 is for smoke tests. +An image encoder trained from scratch is not expected to generalize from a handful of examples. + +When several Idol Gaussians occupy a cell, the cache retains the highest-opacity one. +This is **lossy compression**, not a fidelity-preserving replacement for the full teacher. It can lose fine geometry, +transparency and colour. Teacher colours outside display RGB are clipped for student supervision. + +Training uses balanced cell-occupancy BCE, cell-scaled position error, colour error, log-scale error, +quaternion sign-invariant rotation error, opacity error, and optional geometry occupancy BCE. +It uses direct cached-target regression; no CUDA rasterizer is needed for student training. +The preview renderer is an orthographic CPU diagnostic renderer, not a calibrated quality evaluator. + +The model's speed comes from its small one-pass architecture, not from copying InstantMesh's speed through a loss. +Teacher-level quality, real-photo reconstruction, mesh topology and 1-bit compression are not established. + +## Reproducibility + +`requirements-tested.txt` records the environment used for verification; `pyproject.toml` specifies installable dependencies. +See `VERIFICATION.md` and `examples/synthetic-smoke/` for the delivered test evidence. +No external GPU or cloud training service was used. Source and Python package downloads did use the internet. + +Upstream references: +[InstantMesh](https://github.com/TencentARC/InstantMesh), +[Hi3D](https://huggingface.co/hi3d-official/Hi3Dv1.0), +[Idol SDK](https://github.com/speridlabs/idol-sdk). diff --git a/SHA256SUMS b/SHA256SUMS new file mode 100644 index 0000000000000000000000000000000000000000..8f4cfe66bb2bfc66f0f9858aa3ec98b7dd4e18ef --- /dev/null +++ b/SHA256SUMS @@ -0,0 +1,57 @@ +73d65b3243e83752f6ce82a0d277ea6b68a5477db62c6cf5cdb2eae3706a552b .gitignore +745360a8340c02c0802b6096d86438eface1206b1d0823638b31647c65fd6b0a CHANGELOG.md +cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30 LICENSE +97f90d4705df2cab9b4e7745e8f91eac89296a6c0efb111aecb7fc23863ac905 NOTICE +8f8237ad374e3a45fc4b66504c30e6e4d671cc0a0f78471c5e758b2bbb64a3e3 README.md +3afc5f958305e31c116b19e64690543a49b1f787547f1d5f53e52958f511ea4a THIRD_PARTY.md +3180b6089fea90fba33dd0e1981e0624bb1c21104aaa8fc5ed7a1cfa02cca983 VERIFICATION.md +6ed18e51bf692e882b9b20318721583a1c586922ad02543d2dadb017332f6a8c docs/ALL_MODELS.md +7d9b6bb366592cb06e7c2fbc8923af10f42dbce9abb3d6a0c165dda40f407aee docs/USAGE.md +d37f6f0e5fa95c20ee3e6c78cce997c159fab59aa17b2cd813a01c7c5df96902 examples/synthetic-smoke/README.txt +653ce4a780fe09b68831e3474ba16afe3f7d2943e950aee3bf8ce0e0dc415894 examples/synthetic-smoke/dataset/images/fixture-000.png +00fe7a4c03fb3ffb5b1dc09ba642017d8204d5b72911550f2834985073781088 examples/synthetic-smoke/dataset/images/fixture-001.png +d4a62f7ce5ac714e9905de9e446f0db250afbbb981227609f576bdc87e43558c examples/synthetic-smoke/dataset/images/fixture-002.png +117cd7e973d94613323e87a538176b6f4b33a74b26e30c52cf7ddb27b699a4f5 examples/synthetic-smoke/dataset/images/fixture-003.png +3570b589229609ede457e552668f5b685bbd7271925d3410212dc0797972c31f examples/synthetic-smoke/dataset/images/fixture-004.png +942f83d89c753632a77915647d4bf98686aebff08bd674c2ee1e4d22eece1a92 examples/synthetic-smoke/dataset/images/fixture-005.png +d03a564f9e44223c8c7ef726ad464dbfbfeed7033d3b37b3bf0af0052c029b29 examples/synthetic-smoke/dataset/images/fixture-006.png +f547698cdd97e60ca8d0eb26ffe0c94660adb8d2a71bf987ac0756ec8c589168 examples/synthetic-smoke/dataset/images/fixture-007.png +8409ee247f4fdf4a2c5b31c883f69f3e7939ab4b4015510028078215b55f25db examples/synthetic-smoke/dataset/images/fixture-008.png +09d1383e8111fea53a0a27380a88a6ff0419e423eef2edc9a53c1893eae0b162 examples/synthetic-smoke/dataset/images/fixture-009.png +e0ee306fb0067c16c209ba040fbc24c92c227c5330d353734d55212ebd9bb34a examples/synthetic-smoke/dataset/images/fixture-010.png +d5eab87569f9134717f2a62985df765f9b9c2baf26238e999f36d7ab26c083c2 examples/synthetic-smoke/dataset/images/fixture-011.png +fd87f70cab28136979f63e8cf31a391b35f0a794c60cb3ace5a9aefd95a89ce2 examples/synthetic-smoke/dataset/manifest.jsonl +14c5faa6a9f3496a77eb8d84d3372c028633c4067c19b34acb2f53500723a84b examples/synthetic-smoke/dataset/targets/fixture-000.npz +2c6a8f9acbf991dd1d71e7968c2ee5aae85e8aea88ed130d6ee5c2917bf337bf examples/synthetic-smoke/dataset/targets/fixture-001.npz +59559a2a77e854efbebcd679f5e36c94b83a74b3948f94fc71040b01bf2812eb examples/synthetic-smoke/dataset/targets/fixture-002.npz +6e02ec7927eb6dc1d3eaab79acb34e5f912c1b44ad0400ee0f88de0ff4ddd4d0 examples/synthetic-smoke/dataset/targets/fixture-003.npz +18f8c84baa16107cb7a90513c6d4747aa07efb86e40e05fce174f73bfebe6bce examples/synthetic-smoke/dataset/targets/fixture-004.npz +865602b33c998cbba335eee7ac404724520bb7fe894b9145ac5674ce34627f88 examples/synthetic-smoke/dataset/targets/fixture-005.npz +978618b66ce7ebaedb6a46d8491b661909009d9ee977ddb3a833b27361419001 examples/synthetic-smoke/dataset/targets/fixture-006.npz +6fbd33248e2b1980d091c2b090cd9055a74f241cad63bc1385630374dafbfda1 examples/synthetic-smoke/dataset/targets/fixture-007.npz +814a7856c7013f5c8ddb8b0924aefb74952ab1edee3096b8faf75f48fbf1d3c4 examples/synthetic-smoke/dataset/targets/fixture-008.npz +b1d5e220731f5f07f7e1f92ce13fd9acb7e89451a094dff1c539c9523d33b7b0 examples/synthetic-smoke/dataset/targets/fixture-009.npz +b2501f9ba86b83f1890d845c513b5b0dfe1683b7cf427eb81f0bc793dc156660 examples/synthetic-smoke/dataset/targets/fixture-010.npz +cf657e99a12a1fe9d3e232fea187a1128849644dd145dc154f844af581f1a235 examples/synthetic-smoke/dataset/targets/fixture-011.npz +179e0d93c4e3fd563c4b77f847b8b452d0da57f97a1970c041e9d576e4f7adb9 examples/synthetic-smoke/metrics.jsonl +170aa76fc49fd9486fdaa86e504a2dc56a855a1a6fe6fdadc1aadef1eca072e2 examples/synthetic-smoke/run.json +715a1d0583254513ad7922fb78d906afae454f2b67be4729ebff723643ddb3ad examples/synthetic-smoke/student.json +333cea620e0c898ed1c2de686842d2dacb6fabb4709d13f67b290d3e77fc2430 examples/synthetic-smoke/student.ply +dd1a1719541c5603125e7f9379b0ad8ede20b7e9f2816a04e9120244cdbf528e examples/synthetic-smoke/synthetic-only.pt +94b812437c181ed7d2de1eea5283c6f813745db74f24d3a73422d11d49217904 examples/synthetic-smoke/tests.txt +3a265261c9b999ae5e69a07faa6b30bd894bb8383482ee54440599236810c799 examples/synthetic-smoke/turntable.png +cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30 licenses/Idol-APACHE-2.0.txt +bc8e9ef80c84c150f9768d0caa5c1d2302267ac98f8193549c73459ae0f8c2c8 pyproject.toml +ca8882c02bbaf27c4a1c1ee7c050ffa0fd2146466b9572ca25c008acdf11667a requirements-tested.txt +3ba5d0afbcf107c5c17e8d9e26b4ccb37bb5c8fec7c9efe5fe3eead58be0c2d6 run-local.sh +66016ae8d5b10bde70460d4074a17bddbd357780e4684e9e7ef852c05e65de49 setup.sh +fb16dba6d7b1fe9d7c63703dfa07f9ba143cc3a24249cdfae6aa86197c22a3a1 src/image3d_student/__init__.py +d2066dcc48657504460ebf17e0095dbca61eaedd5a0bdd6a99028c87c64563c6 src/image3d_student/cli.py +d34f9fa50d574764e5e2c1c31c1ca477f2bc4a1887e103c5c47b754f1f1d8c72 src/image3d_student/data.py +e01d1b42929c993d92cad1bde8944b14c6a8e325fc279d8fd3e9c7287523ad69 src/image3d_student/demo.py +cf9a610c56faa21a794df111d5ea1c26ba4afe64d307bdadda2eaac9e82aa8de src/image3d_student/engine.py +a18005f75a66846446b2574b4f1b085ebde5321e835c4e5b4507e15e812ee5ef src/image3d_student/model.py +41961fae62d2819c1e3f455e74539a66b8c4f57bedd9fb9379497b3a5764c692 src/image3d_student/splats.py +2e2439bc0a48c52e32a8e273f879ed7328fdb82632a996ed3e239e0391ca3a59 src/image3d_student/teacher_idol.py +8570a8b6bc79150003eab28c9db117ce3fc7eb6f2cea1d357bdf67e759b541fe tests/test_mps.py +b9970d623f102e6a88908b425da846700af501b7e3fb976e70d00d8684ad6a1b tests/test_pipeline.py diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md new file mode 100644 index 0000000000000000000000000000000000000000..5054e281bd9e736f299974492eaa48169e266d23 --- /dev/null +++ b/THIRD_PARTY.md @@ -0,0 +1,18 @@ +# License scope and external components + +The Apache-2.0 license covers this project's original source, documentation, procedural data and synthetic-only checkpoint. +It does not replace the terms of third-party packages, pretrained teachers, or separately downloaded encoders. + +| Component | Treatment | +|---|---| +| Idol SDK | Optional dependency pinned to `4702a7f00850e34b6e83a2170cc123c3e9570744`; Apache-2.0 attribution retained | +| InstantMesh | External teacher reference/import; no upstream weights or implementation bundled | +| Hi3D v1.0 | External teacher reference/import; its custom license is not replaced by this license | +| DINOv3 / FLUX.2-dev VAE | Gated, separately downloaded encoder dependencies with their own terms | +| PyTorch, NumPy, Pillow, SciPy, trimesh, plyfile | Installed dependencies; each retains its own license | + +The supplied checkpoint was trained on generated ellipsoid fixtures only. It contains no distilled Idol, +InstantMesh or Hi3D training. A future teacher-trained checkpoint requires its own provenance and license assessment. + +Sources: [Idol SDK license](https://github.com/speridlabs/idol-sdk/blob/4702a7f00850e34b6e83a2170cc123c3e9570744/LICENSE), +[Hi3D](https://huggingface.co/hi3d-official/Hi3Dv1.0), [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0). diff --git a/VERIFICATION.md b/VERIFICATION.md new file mode 100644 index 0000000000000000000000000000000000000000..84c47f98d66ced9616643231c42da0e77d09e09c --- /dev/null +++ b/VERIFICATION.md @@ -0,0 +1,40 @@ +# Verification — 1 November 2026 + +## Verified locally + +- Apple Silicon, Python 3.13.18, PyTorch 2.15.0. +- Metal GPU tensor operations and backward pass work outside the app's filesystem sandbox. +- 10 CPU tests passed: Gaussian PLY round trip; mesh/point PLY rejection; coordinate bounds; + deterministic cell selection; cache hash and overwrite protections; aligned geometry target import; + missing-teacher masking; quaternion sign invariance; query-chunk attention equivalence; + object-level split integrity; and measurable loss reduction during training. +- 2 additional Metal tests passed: default 16³ student forward/backward; and the official Idol + architecture's tiny randomly initialized denoiser/sampler, octree and Gaussian decoder. + The tiny teacher uses the pinned upstream SDK with the real-valued RoPE / chunked attention adapter. + +## End-to-end synthetic smoke test + +- 12 procedural ellipsoid image/splat pairs. **No teacher-generated samples.** +- 10 training objects, 2 held-out validation objects; seed 42. +- Student: grid 8, width 64, 306,321 parameters, FP32. +- 100 AdamW updates on MPS, batch size 1, learning rate 0.001. +- Held-out loss: 1.35705 at step 1 → 0.55299 at step 100. +- Held-out occupancy IoU at step 100: 0.48258. +- Timed training loop including periodic validation/checkpointing: 6.383 seconds. + This tiny 8³ fixture test is not a runtime estimate for teacher inference or serious training. +- Reloaded the saved checkpoint and generated a 40-Gaussian PLY for held-out fixture 007. +- Inspected the four-view diagnostic PNG. It is a coarse, sparse synthetic result, not a realistic reconstruction. + +## Not verified / not completed + +- No pretrained Idol weights or gated encoder weights were downloaded. +- No complete real-image Idol inference on MPS; tiny component success does not establish full-size memory feasibility. +- No InstantMesh or Hi3D inference on Mac. Their outputs can only be imported into the student dataset. +- No training on real teacher outputs, real-image evaluation, teacher-quality matching, 1-bit weights, or binary kernels. +- No multiview rendering loss, automatic cross-teacher alignment, or teacher disagreement filtering. + +At inspection time, only about 7.4 GB of disk was available before installing dependencies. +The teacher downloader uses a conservative 16 GiB free-space guard for new downloads. +The user still needs access to DINOv3 and FLUX.2-dev encoder repositories and actual object images/teacher data. + +The files in `examples/synthetic-smoke` are included for reproducibility, not as pretrained production assets. diff --git a/docs/ALL_MODELS.md b/docs/ALL_MODELS.md new file mode 100644 index 0000000000000000000000000000000000000000..2b00399c28cb108f692175cac5cef42ddffdf8d6 --- /dev/null +++ b/docs/ALL_MODELS.md @@ -0,0 +1,34 @@ +# All Lucasgitguy-co model instructions + +Updated 2026-11-02: 13 existing model repositories plus the new Image3D Student first release. +Each guide includes installation, exact artifact names, inference steps or explicit platform limitations, and source references. +Documentation coverage does not imply that every model was executed on every OS. + +| Model | Step-by-step guide | Weight/project license | Platform status | +|---|---|---|---| +| [image3d-student](https://huggingface.co/Lucasgitguy-co/image3d-student) | [Instructions](USAGE.md) | Apache-2.0 | CPU on all three; Metal verified on macOS; synthetic prototype | +| [lfm3-vl-2b-heretic-gguf](https://huggingface.co/Lucasgitguy-co/lfm3-vl-2b-heretic-gguf) | [Instructions](https://huggingface.co/Lucasgitguy-co/lfm3-vl-2b-heretic-gguf/blob/main/docs/USAGE.md) | LFM Open License v1.0 | Windows / Linux / macOS recipes | +| [lfm3-vl-2b-heretic](https://huggingface.co/Lucasgitguy-co/lfm3-vl-2b-heretic) | [Instructions](https://huggingface.co/Lucasgitguy-co/lfm3-vl-2b-heretic/blob/main/docs/USAGE.md) | LFM Open License v1.0 | Windows / Linux / macOS recipes | +| [minimind-3v-moe-adapter](https://huggingface.co/Lucasgitguy-co/minimind-3v-moe-adapter) | [Instructions](https://huggingface.co/Lucasgitguy-co/minimind-3v-moe-adapter/blob/main/docs/USAGE.md) | Apache-2.0 | Base candidate documented; exact original revision unresolved | +| [minicpm-v-4-7-1b-mermaid-adapter](https://huggingface.co/Lucasgitguy-co/minicpm-v-4-7-1b-mermaid-adapter) | [Instructions](https://huggingface.co/Lucasgitguy-co/minicpm-v-4-7-1b-mermaid-adapter/blob/main/docs/USAGE.md) | Apache-2.0 | Windows / Linux / macOS recipes | +| [lfm3-vl-2b-heretic-mlx-4bit](https://huggingface.co/Lucasgitguy-co/lfm3-vl-2b-heretic-mlx-4bit) | [Instructions](https://huggingface.co/Lucasgitguy-co/lfm3-vl-2b-heretic-mlx-4bit/blob/main/docs/USAGE.md) | LFM Open License v1.0 | Apple Silicon recipe; no validated Windows/Linux execution for this artifact | +| [minimind-mermaid-run2](https://huggingface.co/Lucasgitguy-co/minimind-mermaid-run2) | [Instructions](https://huggingface.co/Lucasgitguy-co/minimind-mermaid-run2/blob/main/docs/USAGE.md) | Apache-2.0 | Windows / Linux / macOS recipes | +| [minicpm-mermaid-heretic](https://huggingface.co/Lucasgitguy-co/minicpm-mermaid-heretic) | [Instructions](https://huggingface.co/Lucasgitguy-co/minicpm-mermaid-heretic/blob/main/docs/USAGE.md) | Apache-2.0 | Windows / Linux / macOS recipes | +| [minicpm-mermaid-heretic-trial65](https://huggingface.co/Lucasgitguy-co/minicpm-mermaid-heretic-trial65) | [Instructions](https://huggingface.co/Lucasgitguy-co/minicpm-mermaid-heretic-trial65/blob/main/docs/USAGE.md) | Apache-2.0 | Windows / Linux / macOS recipes | +| [minicpm-mermaid-gguf](https://huggingface.co/Lucasgitguy-co/minicpm-mermaid-gguf) | [Instructions](https://huggingface.co/Lucasgitguy-co/minicpm-mermaid-gguf/blob/main/docs/USAGE.md) | Apache-2.0 | Windows / Linux / macOS recipes | +| [minicpm-mermaid-heretic-gguf](https://huggingface.co/Lucasgitguy-co/minicpm-mermaid-heretic-gguf) | [Instructions](https://huggingface.co/Lucasgitguy-co/minicpm-mermaid-heretic-gguf/blob/main/docs/USAGE.md) | Apache-2.0 | Windows / Linux / macOS recipes | +| [minicpm-mermaid-heretic-trial65-gguf](https://huggingface.co/Lucasgitguy-co/minicpm-mermaid-heretic-trial65-gguf) | [Instructions](https://huggingface.co/Lucasgitguy-co/minicpm-mermaid-heretic-trial65-gguf/blob/main/docs/USAGE.md) | Apache-2.0 | Windows / Linux / macOS recipes | +| [Lucas-Image-2.1-Turbo-Q2K-SakuraEncoder](https://huggingface.co/Lucasgitguy-co/Lucas-Image-2.1-Turbo-Q2K-SakuraEncoder) | [Instructions](https://huggingface.co/Lucasgitguy-co/Lucas-Image-2.1-Turbo-Q2K-SakuraEncoder/blob/main/docs/USAGE.md) | Qwen Research License | Windows / Linux / macOS recipes | +| [Qwen3.5-2B-Mermaid-MLX-LoRA](https://huggingface.co/Lucasgitguy-co/Qwen3.5-2B-Mermaid-MLX-LoRA) | [Instructions](https://huggingface.co/Lucasgitguy-co/Qwen3.5-2B-Mermaid-MLX-LoRA/blob/main/docs/USAGE.md) | Apache-2.0 | Apple Silicon recipe; no validated Windows/Linux execution for this artifact | + +## Related datasets + +The account also contains [mermaid-diagram-text-images](https://huggingface.co/datasets/Lucasgitguy-co/mermaid-diagram-text-images) +and [mermaid-text-en-nl](https://huggingface.co/datasets/Lucasgitguy-co/mermaid-text-en-nl). These are datasets, not runnable model modules. +Their dataset cards describe the data and retain their own terms. Do not combine them into training/evaluation without checking for overlap. + +## License boundaries + +The new Image3D Student original code and synthetic-only artifacts use Apache-2.0. Existing LFM weight releases retain +LFM Open License v1.0; the Qwen image bundle retains the Qwen Research License. Those inherited terms cannot be replaced +by labelling the repositories Apache-2.0. Other existing license declarations were preserved. diff --git a/docs/USAGE.md b/docs/USAGE.md new file mode 100644 index 0000000000000000000000000000000000000000..54eefc4a48347fdc1fc48dd336c78649d84fa352 --- /dev/null +++ b/docs/USAGE.md @@ -0,0 +1,120 @@ +# Image3D Student v0.1.0 — Windows, Linux and macOS + +This is the first public prototype release. The bundled checkpoint learned only synthetic ellipsoids. +It is useful for testing installation, loading, training and PLY export; it is not a general image-to-3D model. +Original project code, documentation and synthetic demonstration artifacts are Apache-2.0. + +Install Python 3.13 (or use 3.11/3.12). Start in a new directory. Windows uses PowerShell. +CPU is supported by the code on all three platforms; actual execution was verified on macOS CPU and Metal only. +This version does not implement a CUDA student device. Do not pass `--device cuda`. + +## Windows + +1. Create an environment and install the Hub downloader. + +```powershell +py -3.13 -m venv .venv +.\.venv\Scripts\python.exe -m pip install --upgrade pip +.\.venv\Scripts\python.exe -m pip install huggingface_hub +``` + +2. Download the immutable first release and install its source package. + +```powershell +.\.venv\Scripts\hf.exe download Lucasgitguy-co/image3d-student --revision v0.1.0 --local-dir image3d-student +.\.venv\Scripts\python.exe -m pip install -e "./image3d-student[test]" +.\.venv\Scripts\python.exe -m image3d_student.cli doctor +``` + +3. Generate a test PLY using the bundled synthetic-only checkpoint. + +```powershell +.\.venv\Scripts\python.exe -m image3d_student.cli generate --checkpoint image3d-student/examples/synthetic-smoke/synthetic-only.pt --image image3d-student/examples/synthetic-smoke/dataset/images/fixture-007.png --out results/synthetic-demo.ply --device cpu +.\.venv\Scripts\python.exe -m image3d_student.cli preview --ply results/synthetic-demo.ply --out results/turntable.png +``` + +4. Test training from scratch on procedural fixtures. + +```powershell +.\.venv\Scripts\python.exe -m image3d_student.cli demo-data --out data/demo --count 12 --grid 8 +.\.venv\Scripts\python.exe -m image3d_student.cli train --manifest data/demo/manifest.jsonl --out runs/demo --device cpu --width 64 --steps 100 --lr 0.001 --eval-every 25 --allow-synthetic +.\.venv\Scripts\python.exe -m pytest image3d-student/tests -q +``` +## Linux + +1. Create an environment and install the Hub downloader. + +```bash +python3.13 -m venv .venv +.venv/bin/python -m pip install --upgrade pip +.venv/bin/python -m pip install huggingface_hub +``` + +2. Download the immutable first release and install its source package. + +```bash +.venv/bin/hf download Lucasgitguy-co/image3d-student --revision v0.1.0 --local-dir image3d-student +.venv/bin/python -m pip install -e "./image3d-student[test]" +.venv/bin/python -m image3d_student.cli doctor +``` + +3. Generate a test PLY using the bundled synthetic-only checkpoint. + +```bash +.venv/bin/python -m image3d_student.cli generate --checkpoint image3d-student/examples/synthetic-smoke/synthetic-only.pt --image image3d-student/examples/synthetic-smoke/dataset/images/fixture-007.png --out results/synthetic-demo.ply --device cpu +.venv/bin/python -m image3d_student.cli preview --ply results/synthetic-demo.ply --out results/turntable.png +``` + +4. Test training from scratch on procedural fixtures. + +```bash +.venv/bin/python -m image3d_student.cli demo-data --out data/demo --count 12 --grid 8 +.venv/bin/python -m image3d_student.cli train --manifest data/demo/manifest.jsonl --out runs/demo --device cpu --width 64 --steps 100 --lr 0.001 --eval-every 25 --allow-synthetic +.venv/bin/python -m pytest image3d-student/tests -q +``` +## macOS + +1. Create an environment and install the Hub downloader. + +```bash +python3.13 -m venv .venv +.venv/bin/python -m pip install --upgrade pip +.venv/bin/python -m pip install huggingface_hub +``` + +2. Download the immutable first release and install its source package. + +```bash +.venv/bin/hf download Lucasgitguy-co/image3d-student --revision v0.1.0 --local-dir image3d-student +.venv/bin/python -m pip install -e "./image3d-student[test]" +.venv/bin/python -m image3d_student.cli doctor +``` + +3. Generate a test PLY using the bundled synthetic-only checkpoint. + +```bash +.venv/bin/python -m image3d_student.cli generate --checkpoint image3d-student/examples/synthetic-smoke/synthetic-only.pt --image image3d-student/examples/synthetic-smoke/dataset/images/fixture-007.png --out results/synthetic-demo.ply --device mps +.venv/bin/python -m image3d_student.cli preview --ply results/synthetic-demo.ply --out results/turntable.png +``` + +4. Test training from scratch on procedural fixtures. + +```bash +.venv/bin/python -m image3d_student.cli demo-data --out data/demo --count 12 --grid 8 +.venv/bin/python -m image3d_student.cli train --manifest data/demo/manifest.jsonl --out runs/demo --device mps --width 64 --steps 100 --lr 0.001 --eval-every 25 --allow-synthetic +.venv/bin/python -m pytest image3d-student/tests -q +``` + +On Intel Macs, or when Metal is unavailable, replace `--device mps` with `--device cpu`. + +## Train on real teacher outputs + +Use the [project workflow](../README.md#train-with-real-teacher-data) to cache the exact teacher input image and Idol PLY, optionally adding explicitly aligned InstantMesh/Hi3D meshes. Use a new run/output directory for each experiment. These models are not downloaded or run by the student training command. + +Only the experimental Idol runner exists in this release. Full pretrained Idol inference remains unverified; InstantMesh/Hi3D are import-only integrations. The Mac adapter is not a validated Windows/Linux teacher runner. All upstream model/encoder licenses and access terms remain separate from this prototype's Apache license. + +## Version and outputs + +`v0.1.0` pins this first release. Downloading `main` instead may fetch later changes. The `.ply` file is a Gaussian-splat artifact, not a triangle mesh. `last.pt` is the latest training checkpoint; `best.pt` exists only when held-out validation is enabled. Outputs refuse overwriting: choose a new filename/directory when rerunning. + +No 1-bit quantization is implemented. The default untrained architecture has 465,041 parameters; the bundled smaller smoke-test checkpoint has 306,321 parameters. See [verification](../VERIFICATION.md) for the exact test conditions. diff --git a/examples/synthetic-smoke/README.txt b/examples/synthetic-smoke/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..3c3de9ca27706c1cc740e0584ab510528497a0b1 --- /dev/null +++ b/examples/synthetic-smoke/README.txt @@ -0,0 +1,5 @@ +SYNTHETIC FIXTURES ONLY. No Idol, InstantMesh or Hi3D data was used. +The checkpoint has only learned 10 procedural ellipsoid objects. +See ../../VERIFICATION.md. +To reproduce inference from the project directory: +bash run-local.sh generate --checkpoint examples/synthetic-smoke/synthetic-only.pt --image examples/synthetic-smoke/dataset/images/fixture-007.png --out results/demo.ply --device mps diff --git a/examples/synthetic-smoke/dataset/images/fixture-000.png b/examples/synthetic-smoke/dataset/images/fixture-000.png new file mode 100644 index 0000000000000000000000000000000000000000..72434538fd60e3ae7536279eb282cff357eb86ea Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-000.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-001.png b/examples/synthetic-smoke/dataset/images/fixture-001.png new file mode 100644 index 0000000000000000000000000000000000000000..57b88891136fc77db32d710a6ea665036e02fb70 Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-001.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-002.png b/examples/synthetic-smoke/dataset/images/fixture-002.png new file mode 100644 index 0000000000000000000000000000000000000000..95896e53cd033012dffc6d9eec1faf1113076845 Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-002.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-003.png b/examples/synthetic-smoke/dataset/images/fixture-003.png new file mode 100644 index 0000000000000000000000000000000000000000..9715cc9d9065c790a03ef422ba2d67e0a6326db9 Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-003.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-004.png b/examples/synthetic-smoke/dataset/images/fixture-004.png new file mode 100644 index 0000000000000000000000000000000000000000..44c37df53b8ce682694446e53b53d13d8caa39c8 Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-004.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-005.png b/examples/synthetic-smoke/dataset/images/fixture-005.png new file mode 100644 index 0000000000000000000000000000000000000000..e4395d39af135e1b641c8a9c93b27f19ecb0368b Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-005.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-006.png b/examples/synthetic-smoke/dataset/images/fixture-006.png new file mode 100644 index 0000000000000000000000000000000000000000..eb776dab77154b944336d70446ba494fc9c4d927 Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-006.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-007.png b/examples/synthetic-smoke/dataset/images/fixture-007.png new file mode 100644 index 0000000000000000000000000000000000000000..25c82cd613b4062995a9eb19db2f3ffc77644700 Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-007.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-008.png b/examples/synthetic-smoke/dataset/images/fixture-008.png new file mode 100644 index 0000000000000000000000000000000000000000..8afc4c84b3b26569cbd4ecbf4ba35e267521a917 Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-008.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-009.png b/examples/synthetic-smoke/dataset/images/fixture-009.png new file mode 100644 index 0000000000000000000000000000000000000000..5d27743e530dee7cfac3c6fa25ff7701f76d524b Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-009.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-010.png b/examples/synthetic-smoke/dataset/images/fixture-010.png new file mode 100644 index 0000000000000000000000000000000000000000..3f94d523b5903d9b966e781d467debbc14a2f260 Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-010.png differ diff --git a/examples/synthetic-smoke/dataset/images/fixture-011.png b/examples/synthetic-smoke/dataset/images/fixture-011.png new file mode 100644 index 0000000000000000000000000000000000000000..95523e6941bd4624021d982a54594bcd41c27504 Binary files /dev/null and b/examples/synthetic-smoke/dataset/images/fixture-011.png differ diff --git a/examples/synthetic-smoke/dataset/manifest.jsonl b/examples/synthetic-smoke/dataset/manifest.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0b2f1ba138b1e1f9ab501e6b7a34b42428431363 --- /dev/null +++ b/examples/synthetic-smoke/dataset/manifest.jsonl @@ -0,0 +1,12 @@ +{"id": "fixture-000", "image": "images/fixture-000.png", "target": "targets/fixture-000.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 80, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-000"}, "image_sha256": "653ce4a780fe09b68831e3474ba16afe3f7d2943e950aee3bf8ce0e0dc415894", "target_sha256": "14c5faa6a9f3496a77eb8d84d3372c028633c4067c19b34acb2f53500723a84b"} +{"id": "fixture-001", "image": "images/fixture-001.png", "target": "targets/fixture-001.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 72, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-001"}, "image_sha256": "00fe7a4c03fb3ffb5b1dc09ba642017d8204d5b72911550f2834985073781088", "target_sha256": "2c6a8f9acbf991dd1d71e7968c2ee5aae85e8aea88ed130d6ee5c2917bf337bf"} +{"id": "fixture-002", "image": "images/fixture-002.png", "target": "targets/fixture-002.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 76, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-002"}, "image_sha256": "d4a62f7ce5ac714e9905de9e446f0db250afbbb981227609f576bdc87e43558c", "target_sha256": "59559a2a77e854efbebcd679f5e36c94b83a74b3948f94fc71040b01bf2812eb"} +{"id": "fixture-003", "image": "images/fixture-003.png", "target": "targets/fixture-003.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 40, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-003"}, "image_sha256": "117cd7e973d94613323e87a538176b6f4b33a74b26e30c52cf7ddb27b699a4f5", "target_sha256": "6e02ec7927eb6dc1d3eaab79acb34e5f912c1b44ad0400ee0f88de0ff4ddd4d0"} +{"id": "fixture-004", "image": "images/fixture-004.png", "target": "targets/fixture-004.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 80, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-004"}, "image_sha256": "3570b589229609ede457e552668f5b685bbd7271925d3410212dc0797972c31f", "target_sha256": "18f8c84baa16107cb7a90513c6d4747aa07efb86e40e05fce174f73bfebe6bce"} +{"id": "fixture-005", "image": "images/fixture-005.png", "target": "targets/fixture-005.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 80, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-005"}, "image_sha256": "942f83d89c753632a77915647d4bf98686aebff08bd674c2ee1e4d22eece1a92", "target_sha256": "865602b33c998cbba335eee7ac404724520bb7fe894b9145ac5674ce34627f88"} +{"id": "fixture-006", "image": "images/fixture-006.png", "target": "targets/fixture-006.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 64, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-006"}, "image_sha256": "d03a564f9e44223c8c7ef726ad464dbfbfeed7033d3b37b3bf0af0052c029b29", "target_sha256": "978618b66ce7ebaedb6a46d8491b661909009d9ee977ddb3a833b27361419001"} +{"id": "fixture-007", "image": "images/fixture-007.png", "target": "targets/fixture-007.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 48, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-007"}, "image_sha256": "f547698cdd97e60ca8d0eb26ffe0c94660adb8d2a71bf987ac0756ec8c589168", "target_sha256": "6fbd33248e2b1980d091c2b090cd9055a74f241cad63bc1385630374dafbfda1"} +{"id": "fixture-008", "image": "images/fixture-008.png", "target": "targets/fixture-008.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 84, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-008"}, "image_sha256": "8409ee247f4fdf4a2c5b31c883f69f3e7939ab4b4015510028078215b55f25db", "target_sha256": "814a7856c7013f5c8ddb8b0924aefb74952ab1edee3096b8faf75f48fbf1d3c4"} +{"id": "fixture-009", "image": "images/fixture-009.png", "target": "targets/fixture-009.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 64, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-009"}, "image_sha256": "09d1383e8111fea53a0a27380a88a6ff0419e423eef2edc9a53c1893eae0b162", "target_sha256": "b1d5e220731f5f07f7e1f92ce13fd9acb7e89451a094dff1c539c9523d33b7b0"} +{"id": "fixture-010", "image": "images/fixture-010.png", "target": "targets/fixture-010.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 40, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-010"}, "image_sha256": "e0ee306fb0067c16c209ba040fbc24c92c227c5330d353734d55212ebd9bb34a", "target_sha256": "b2501f9ba86b83f1890d845c513b5b0dfe1683b7cf427eb81f0bc793dc156660"} +{"id": "fixture-011", "image": "images/fixture-011.png", "target": "targets/fixture-011.npz", "source": "synthetic", "grid": 8, "teachers": {"appearance": "synthetic", "coarse": null, "fine": null}, "teacher_gaussians": 392, "retained_gaussians": 72, "geometry_transforms": {"coarse": null, "fine": null}, "metadata": {"purpose": "code verification only", "object_id": "fixture-011"}, "image_sha256": "d5eab87569f9134717f2a62985df765f9b9c2baf26238e999f36d7ab26c083c2", "target_sha256": "cf657e99a12a1fe9d3e232fea187a1128849644dd145dc154f844af581f1a235"} diff --git a/examples/synthetic-smoke/dataset/targets/fixture-000.npz b/examples/synthetic-smoke/dataset/targets/fixture-000.npz new file mode 100644 index 0000000000000000000000000000000000000000..4dc0b127e818e275a4c2989a8a286fa7837dda11 --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-000.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14c5faa6a9f3496a77eb8d84d3372c028633c4067c19b34acb2f53500723a84b +size 3732 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-001.npz b/examples/synthetic-smoke/dataset/targets/fixture-001.npz new file mode 100644 index 0000000000000000000000000000000000000000..7fffb56479105832becf4983f3daf42a6f8c749b --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-001.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c6a8f9acbf991dd1d71e7968c2ee5aae85e8aea88ed130d6ee5c2917bf337bf +size 3701 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-002.npz b/examples/synthetic-smoke/dataset/targets/fixture-002.npz new file mode 100644 index 0000000000000000000000000000000000000000..41749d49a966e8160c491103a4481527da5e9cc0 --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-002.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59559a2a77e854efbebcd679f5e36c94b83a74b3948f94fc71040b01bf2812eb +size 3732 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-003.npz b/examples/synthetic-smoke/dataset/targets/fixture-003.npz new file mode 100644 index 0000000000000000000000000000000000000000..fd8966b6d9a6aa0b55319d66b2cdca92a3c8dd42 --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-003.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e02ec7927eb6dc1d3eaab79acb34e5f912c1b44ad0400ee0f88de0ff4ddd4d0 +size 3431 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-004.npz b/examples/synthetic-smoke/dataset/targets/fixture-004.npz new file mode 100644 index 0000000000000000000000000000000000000000..d984d279800866bee0388d6071c66478c541e87b --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-004.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18f8c84baa16107cb7a90513c6d4747aa07efb86e40e05fce174f73bfebe6bce +size 3750 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-005.npz b/examples/synthetic-smoke/dataset/targets/fixture-005.npz new file mode 100644 index 0000000000000000000000000000000000000000..4705de3697f3baae908317faa6808b27fa688f27 --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-005.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:865602b33c998cbba335eee7ac404724520bb7fe894b9145ac5674ce34627f88 +size 3728 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-006.npz b/examples/synthetic-smoke/dataset/targets/fixture-006.npz new file mode 100644 index 0000000000000000000000000000000000000000..002bc24369c7dfc3dde0424a221d88e6d248e645 --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-006.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:978618b66ce7ebaedb6a46d8491b661909009d9ee977ddb3a833b27361419001 +size 3746 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-007.npz b/examples/synthetic-smoke/dataset/targets/fixture-007.npz new file mode 100644 index 0000000000000000000000000000000000000000..be1b9cef6960384c1a3c8f87720aabe2b0c7275b --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-007.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fbd33248e2b1980d091c2b090cd9055a74f241cad63bc1385630374dafbfda1 +size 3554 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-008.npz b/examples/synthetic-smoke/dataset/targets/fixture-008.npz new file mode 100644 index 0000000000000000000000000000000000000000..e0bc4148feadb0e6c7d6dd0553625fe36a672596 --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-008.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:814a7856c7013f5c8ddb8b0924aefb74952ab1edee3096b8faf75f48fbf1d3c4 +size 3814 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-009.npz b/examples/synthetic-smoke/dataset/targets/fixture-009.npz new file mode 100644 index 0000000000000000000000000000000000000000..bbdae3275c22feabab5d1819daca24b839515daf --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-009.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1d5e220731f5f07f7e1f92ce13fd9acb7e89451a094dff1c539c9523d33b7b0 +size 3670 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-010.npz b/examples/synthetic-smoke/dataset/targets/fixture-010.npz new file mode 100644 index 0000000000000000000000000000000000000000..ce5ce93cc82c8a96465f49656366cc5169bf17ae --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-010.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2501f9ba86b83f1890d845c513b5b0dfe1683b7cf427eb81f0bc793dc156660 +size 3437 diff --git a/examples/synthetic-smoke/dataset/targets/fixture-011.npz b/examples/synthetic-smoke/dataset/targets/fixture-011.npz new file mode 100644 index 0000000000000000000000000000000000000000..d8a69fc9ff3ed0bc82ca09d1b10c529ae0079a5a --- /dev/null +++ b/examples/synthetic-smoke/dataset/targets/fixture-011.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf657e99a12a1fe9d3e232fea187a1128849644dd145dc154f844af581f1a235 +size 3761 diff --git a/examples/synthetic-smoke/metrics.jsonl b/examples/synthetic-smoke/metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fc981df36580aed3c7e61160c8727cabc7f2e869 --- /dev/null +++ b/examples/synthetic-smoke/metrics.jsonl @@ -0,0 +1,5 @@ +{"step": 1, "seconds": 4.36, "train_loss": 1.446029543876648, "occupancy": 1.1178711652755737, "position": 0.09457678347826004, "colour": 0.03700888156890869, "scale": 0.021625641733407974, "rotation": 0.0015759735833853483, "opacity": 0.15724360942840576, "geometry": -0.0, "validation": {"occupancy": 1.0542445182800293, "position": 0.11266224458813667, "colour": 0.02881160657852888, "scale": 0.021875507198274136, "rotation": 0.011832301504909992, "opacity": 0.1291448101401329, "geometry": 0.0, "loss": 1.357045590877533, "occupancy_iou": 0.0}} +{"step": 25, "seconds": 4.899, "train_loss": 0.8945547938346863, "occupancy": 0.6847372651100159, "position": 0.11709830909967422, "colour": 0.04580960050225258, "scale": 0.0055952477268874645, "rotation": 0.004777997732162476, "opacity": 6.272108294069767e-05, "geometry": 0.0, "validation": {"occupancy": 0.6845944821834564, "position": 0.10255199298262596, "colour": 0.01412782957777381, "scale": 0.008690122980624437, "rotation": 0.00476150750182569, "opacity": 5.126132509758463e-05, "geometry": 0.0, "loss": 0.8167985677719116, "occupancy_iou": 0.13314673863351345}} +{"step": 50, "seconds": 5.301, "train_loss": 0.804593563079834, "occupancy": 0.6701697111129761, "position": 0.10525131225585938, "colour": 0.014356697909533978, "scale": 0.0017124137375503778, "rotation": 0.0019470214610919356, "opacity": 9.318323282059282e-05, "geometry": 0.0, "validation": {"occupancy": 0.657372236251831, "position": 0.10001565888524055, "colour": 0.013939401600509882, "scale": 0.0034465506905689836, "rotation": 0.0037114854203537107, "opacity": 8.069136629273999e-05, "geometry": 0.0, "loss": 0.7860631942749023, "occupancy_iou": 0.3602825775742531}} +{"step": 75, "seconds": 5.847, "train_loss": 0.6927299499511719, "occupancy": 0.5821881890296936, "position": 0.08322491496801376, "colour": 0.012623971328139305, "scale": 0.015208232216536999, "rotation": 0.0022862323094159365, "opacity": 0.0003194920136593282, "geometry": 0.0, "validation": {"occupancy": 0.6055956184864044, "position": 0.09585224837064743, "colour": 0.013855834025889635, "scale": 0.005587705178186297, "rotation": 0.004499655216932297, "opacity": 0.0020019400981254876, "geometry": 0.0, "loss": 0.7321702241897583, "occupancy_iou": 0.15090909227728844}} +{"step": 100, "seconds": 6.383, "train_loss": 0.513293981552124, "occupancy": 0.38297760486602783, "position": 0.10490132123231888, "colour": 0.011903365142643452, "scale": 0.01172574982047081, "rotation": 0.00287768617272377, "opacity": 0.00014796240429859608, "geometry": 0.0, "validation": {"occupancy": 0.42350494861602783, "position": 0.0950610339641571, "colour": 0.015541733242571354, "scale": 0.029876035638153553, "rotation": 0.0022482963977381587, "opacity": 0.00012832072752644308, "geometry": 0.0, "loss": 0.5529901683330536, "occupancy_iou": 0.4825790077447891}} diff --git a/examples/synthetic-smoke/run.json b/examples/synthetic-smoke/run.json new file mode 100644 index 0000000000000000000000000000000000000000..3de282e949c8ef145b5b53ddf88cfc7fc34de141 --- /dev/null +++ b/examples/synthetic-smoke/run.json @@ -0,0 +1,29 @@ +{ + "synthetic_only": true, + "contains_synthetic": true, + "device": "mps", + "torch": "2.15.0", + "seed": 42, + "train_ids": [ + "fixture-002", + "fixture-008", + "fixture-009", + "fixture-006", + "fixture-011", + "fixture-003", + "fixture-004", + "fixture-000", + "fixture-001", + "fixture-010" + ], + "val_ids": [ + "fixture-007", + "fixture-005" + ], + "parameters": 306321, + "teachers": [ + "synthetic" + ], + "validation": "held-out objects", + "dataset": "examples/synthetic-smoke/dataset/manifest.jsonl" +} \ No newline at end of file diff --git a/examples/synthetic-smoke/student.json b/examples/synthetic-smoke/student.json new file mode 100644 index 0000000000000000000000000000000000000000..b6606f48d5fc6fd02cf1fc8060bf75b0b602ac24 --- /dev/null +++ b/examples/synthetic-smoke/student.json @@ -0,0 +1,36 @@ +{ + "gaussians": 40, + "seconds_preprocess_and_forward": 1.8782236250117421, + "device": "mps", + "checkpoint_step": 100, + "training": { + "synthetic_only": true, + "contains_synthetic": true, + "device": "mps", + "torch": "2.15.0", + "seed": 42, + "train_ids": [ + "fixture-002", + "fixture-008", + "fixture-009", + "fixture-006", + "fixture-011", + "fixture-003", + "fixture-004", + "fixture-000", + "fixture-001", + "fixture-010" + ], + "val_ids": [ + "fixture-007", + "fixture-005" + ], + "parameters": 306321, + "teachers": [ + "synthetic" + ], + "validation": "held-out objects", + "dataset": "examples/synthetic-smoke/dataset/manifest.jsonl" + }, + "note": "Synthetic-trained output is a pipeline demonstration, not a pretrained image-to-3D result." +} \ No newline at end of file diff --git a/examples/synthetic-smoke/student.ply b/examples/synthetic-smoke/student.ply new file mode 100644 index 0000000000000000000000000000000000000000..6f4b9449d380dcca553e7c2e1da0ca2191664b01 Binary files /dev/null and b/examples/synthetic-smoke/student.ply differ diff --git a/examples/synthetic-smoke/synthetic-only.pt b/examples/synthetic-smoke/synthetic-only.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac50c57f9b5d73d83b2a7fa65a44006ee1481fc3 --- /dev/null +++ b/examples/synthetic-smoke/synthetic-only.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd1a1719541c5603125e7f9379b0ad8ede20b7e9f2816a04e9120244cdbf528e +size 1296253 diff --git a/examples/synthetic-smoke/tests.txt b/examples/synthetic-smoke/tests.txt new file mode 100644 index 0000000000000000000000000000000000000000..ea3f5c29b3afe46dea0642f3225072f334c3b1be --- /dev/null +++ b/examples/synthetic-smoke/tests.txt @@ -0,0 +1,2 @@ +............ [100%] +12 passed in 7.79s diff --git a/examples/synthetic-smoke/turntable.png b/examples/synthetic-smoke/turntable.png new file mode 100644 index 0000000000000000000000000000000000000000..fb63c157c846071c86c764c160fb08011d0670f8 Binary files /dev/null and b/examples/synthetic-smoke/turntable.png differ diff --git a/licenses/Idol-APACHE-2.0.txt b/licenses/Idol-APACHE-2.0.txt new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/licenses/Idol-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..c198ebeb138ad5dddb5e6170974cde21870ff32b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "image3d-student" +version = "0.1.0" +license = "Apache-2.0" +license-files = ["LICENSE", "NOTICE", "THIRD_PARTY.md", "licenses/*.txt"] +readme = "README.md" +authors = [{name = "Lucasgitguy-co"}] +description = "Experimental local image-to-Gaussian student with cached teacher supervision" +requires-python = ">=3.11,<3.14" +dependencies = ["torch>=2.5", "numpy>=1.26", "pillow>=10", "plyfile>=1", "trimesh>=4", "scipy>=1.11"] + +[project.optional-dependencies] +test = ["pytest>=8"] +teacher = ["idol @ git+https://github.com/speridlabs/idol-sdk.git@4702a7f00850e34b6e83a2170cc123c3e9570744"] + +[project.scripts] +image3d = "image3d_student.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/requirements-tested.txt b/requirements-tested.txt new file mode 100644 index 0000000000000000000000000000000000000000..0cae72fe4cfdeda060ab725c291d6569865ceddf --- /dev/null +++ b/requirements-tested.txt @@ -0,0 +1,53 @@ +annotated-doc==0.0.5 +anyio==4.16.0 +certifi==2026.10.23 +charset-normalizer==3.5.2 +click==8.5.0 +cloudpickle==3.1.2 +diffusers==0.42.1 +filelock==4.2.6 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.8.0 +httpcore2==2.16.0 +httpx2==2.16.0 +huggingface-hub==2.4.0 +idna==3.20 +idol @ git+https://github.com/speridlabs/idol-sdk.git@4702a7f00850e34b6e83a2170cc123c3e9570744 +importlib-metadata==10.1.0 +iniconfig==2.3.1 +jinja2==3.1.6 +markdown-it-py==4.2.0 +markupsafe==3.0.4 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.7 +numpy==2.5.4 +packaging==26.3 +pillow==13.0.0 +pluggy==1.7.0 +plyfile==1.1.5 +pygments==2.21.0 +pytest==9.2.0 +pyvers==0.2.3 +pyyaml==6.0.3 +regex==2026.9.29 +requests==2.34.2 +rich==15.0.0 +safetensors==0.8.0 +scipy==1.18.1 +setuptools==84.2.1 +shellingham==1.5.4 +sympy==1.14.0 +tensordict==0.15.4 +tokenizers==0.23.3 +torch==2.15.0 +torchvision==0.29.2 +tqdm==4.70.1 +transformers==5.22.1 +trimesh==5.2.0 +truststore==0.10.5 +typer==0.27.3 +typing-extensions==4.16.0 +urllib3==2.8.0 +zipp==4.1.2 diff --git a/run-local.sh b/run-local.sh new file mode 100644 index 0000000000000000000000000000000000000000..9771a423e9432b432143312af529f862303832ab --- /dev/null +++ b/run-local.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")" +if [ -x .venv/bin/python ]; then + exec .venv/bin/python -m image3d_student.cli "$@" +fi +# Reuse the environment installed during this chat when running in the delivered outputs folder. +if [ -x ../../work/image3d-student/.venv/bin/python ]; then + export PYTHONPATH="$PWD/src${PYTHONPATH:+:$PYTHONPATH}" + exec ../../work/image3d-student/.venv/bin/python -m image3d_student.cli "$@" +fi +echo "No local environment found. Run bash setup.sh first." +exit 1 diff --git a/setup.sh b/setup.sh new file mode 100644 index 0000000000000000000000000000000000000000..ed6966b04ac0e68b5d25724622ba224add77b551 --- /dev/null +++ b/setup.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")" +export UV_CACHE_DIR="$PWD/.cache/uv" +if ! command -v uv >/dev/null 2>&1; then + echo "Install uv or create a Python 3.11–3.13 virtual environment and run: python -m pip install -e '.[test]'" + exit 1 +fi +uv venv --python 3.13 .venv +uv pip install --python .venv/bin/python -e '.[test]' +echo "Ready. Run: source .venv/bin/activate && image3d doctor" diff --git a/src/image3d_student/__init__.py b/src/image3d_student/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..480925728fcf8ac11ed7a3c71c756059d5070632 --- /dev/null +++ b/src/image3d_student/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Lucasgitguy-co +"""Research prototype; no pretrained general-purpose student weights are bundled.""" + +__version__ = "0.1.0" diff --git a/src/image3d_student/cli.py b/src/image3d_student/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..4fbae6e54b3b460abc6dfc23724fa9693b880188 --- /dev/null +++ b/src/image3d_student/cli.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Lucasgitguy-co +import argparse +import importlib.util +import json +import platform +import shutil +import sys +from pathlib import Path + +import torch + + +def parser(): + p = argparse.ArgumentParser(description="Local image-to-Gaussian student research prototype") + sub = p.add_subparsers(dest="command", required=True) + sub.add_parser("doctor", help="Check dependencies, Metal availability and disk space; no downloads") + demo = sub.add_parser("demo-data", help="Create synthetic code-test fixtures, NOT teacher data") + demo.add_argument("--out", required=True) + demo.add_argument("--count", type=int, default=12) + demo.add_argument("--grid", type=int, choices=[8,16,32], default=8) + cache = sub.add_parser("cache", help="Cache Idol PLY with optional aligned geometry teacher meshes") + cache.add_argument("--image", required=True, help="Exact prepared/framed input used by teacher") + cache.add_argument("--idol-ply", required=True) + cache.add_argument("--out", required=True) + cache.add_argument("--id", required=True) + cache.add_argument("--object-id", help="Shared ID for all images of the same object, to prevent validation leakage") + cache.add_argument("--grid", type=int, choices=[8,16,32], default=16) + cache.add_argument("--coarse-mesh", help="InstantMesh output; this command imports, does not run InstantMesh") + cache.add_argument("--coarse-transform", help="JSON 4x4 mesh-to-Idol-PLY matrix") + cache.add_argument("--fine-mesh", help="Hi3D output; this command imports, does not run Hi3D") + cache.add_argument("--fine-transform", help="JSON 4x4 mesh-to-Idol-PLY matrix") + train = sub.add_parser("train", help="Train the student on cached targets") + train.add_argument("--manifest", required=True) + train.add_argument("--out", required=True) + train.add_argument("--device", choices=["auto","mps","cpu"], default="auto") + train.add_argument("--steps", type=int, default=1000) + train.add_argument("--batch-size", type=int, default=1) + train.add_argument("--width", type=int, default=128) + train.add_argument("--lr", type=float, default=3e-4) + train.add_argument("--seed", type=int, default=42) + train.add_argument("--eval-every", type=int, default=50) + train.add_argument("--validation-fraction", type=float, default=.2) + train.add_argument("--allow-synthetic", action="store_true") + generate = sub.add_parser("generate", help="Generate standard 3DGS PLY with a trained student") + generate.add_argument("--checkpoint", required=True) + generate.add_argument("--image", required=True, help="Centered cutout or teacher-prepared image, black background") + generate.add_argument("--out", required=True) + generate.add_argument("--device", choices=["auto","mps","cpu"], default="auto") + generate.add_argument("--threshold", type=float, default=.5) + generate.add_argument("--max-splats", type=int, default=32768) + render = sub.add_parser("preview", help="CPU diagnostic turntable contact sheet") + render.add_argument("--ply", required=True) + render.add_argument("--out", required=True) + teacher = sub.add_parser("teacher-idol", help="EXPERIMENTAL staged Mac runner; weights not bundled") + teacher.add_argument("--image", required=True) + teacher.add_argument("--out", required=True) + teacher.add_argument("--cache", default=".cache/huggingface") + teacher.add_argument("--weights", default="speridlabs/idol") + teacher.add_argument("--device", choices=["mps","cpu"], default="mps") + teacher.add_argument("--mode", choices=["real","synthetic"], default="synthetic") + teacher.add_argument("--steps", type=int, default=20) + teacher.add_argument("--gaussians", type=int, default=32768) + teacher.add_argument("--query-chunk", type=int, default=128) + teacher.add_argument("--seed", type=int, default=42) + teacher.add_argument("--allow-download", action="store_true", help="Explicitly allow downloading model weights and gated encoders") + return p + + +def main(): + p = parser() + args = p.parse_args() + try: + if args.command == "doctor": + report = {"python": platform.python_version(), "platform": platform.platform(), "torch": str(torch.__version__), + "mps_available": torch.backends.mps.is_available(), "mps_built": torch.backends.mps.is_built(), + "free_disk_gib": round(shutil.disk_usage(Path.cwd()).free / 1024**3, 2), + "idol_installed": importlib.util.find_spec("idol") is not None, + "teacher_status": {"Idol": "experimental adapter, real weights unverified", + "InstantMesh": "cached mesh import only; no Mac execution adapter", + "Hi3D": "cached mesh import only; no Mac execution adapter"}, + "weights_bundled": False, "quantization": "FP32 student; no binary kernels or 1-bit claims"} + print(json.dumps(report, indent=2)) + elif args.command == "demo-data": + from .demo import create_demo + print(create_demo(args.out, args.count, args.grid)) + elif args.command == "cache": + from .data import cache_sample + from .splats import read_ply + print(json.dumps(cache_sample(args.out, args.id, args.image, read_ply(args.idol_ply), args.grid, + coarse=args.coarse_mesh, coarse_transform=args.coarse_transform, + fine=args.fine_mesh, fine_transform=args.fine_transform, + extra={"object_id": args.object_id or args.id, "idol_ply": str(Path(args.idol_ply).resolve())}), indent=2)) + elif args.command == "train": + from .engine import train + train(args) + elif args.command == "generate": + from .engine import generate + print(json.dumps(generate(args.checkpoint, args.image, args.out, args.device, args.threshold, args.max_splats), indent=2)) + elif args.command == "preview": + from PIL import Image, ImageDraw + import numpy as np + from .splats import preview, read_ply + splats = read_ply(args.ply) + sheet = Image.new("RGB", (1024, 280), "#111111") + draw = ImageDraw.Draw(sheet) + for i in range(4): + sheet.paste(preview(splats, yaw=i*np.pi/2), (i*256, 24)) + draw.text((i*256+8, 6), f"{i*90} degrees - diagnostic view", fill="white") + out = Path(args.out) + if out.exists(): + raise ValueError("Preview already exists") + out.parent.mkdir(parents=True, exist_ok=True) + sheet.save(out) + elif args.command == "teacher-idol": + from .teacher_idol import run_idol + run_idol(args) + except (ValueError, RuntimeError, FileNotFoundError, ImportError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/image3d_student/data.py b/src/image3d_student/data.py new file mode 100644 index 0000000000000000000000000000000000000000..239da27c8ab17b6954ca366ec9087e8b17ea9fca --- /dev/null +++ b/src/image3d_student/data.py @@ -0,0 +1,149 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Lucasgitguy-co +import hashlib +import json +from pathlib import Path + +import numpy as np +import torch +from PIL import Image, ImageOps +from torch.utils.data import Dataset + +from .splats import Splats, read_ply + + +def grid_centers(grid): + axis = (np.arange(grid, dtype=np.float32) + .5) / grid - .5 + return np.stack(np.meshgrid(axis, axis, axis, indexing="ij"), axis=-1).reshape(-1, 3) + + +def cell_indices(xyz, grid): + if not np.isfinite(xyz).all() or (np.abs(xyz) > .50001).any(): + raise ValueError("Targets must be aligned inside [-0.5,0.5]^3. No silent normalization is performed.") + ijk = np.floor((xyz + .5) * grid).astype(np.int64).clip(0, grid-1) + return (ijk[:, 0] * grid + ijk[:, 1]) * grid + ijk[:, 2] + + +def splat_targets(splats, grid): + """One highest-opacity teacher Gaussian per occupied cell; explicitly lossy.""" + cells = cell_indices(splats.xyz, grid) + order = np.argsort(-splats.opacity[:, 0], kind="stable") + _, first = np.unique(cells[order], return_index=True) + keep = order[first] + slots = cells[keep] + result = {"occ": np.zeros((grid**3, 1), np.float32), "xyz": grid_centers(grid), + "rgb": np.zeros((grid**3, 3), np.float32), + "scale": np.full((grid**3, 3), 1 / grid / 3, np.float32), + "quat": np.tile(np.array([1, 0, 0, 0], np.float32), (grid**3, 1)), + "opacity": np.zeros((grid**3, 1), np.float32), + "geometry": np.zeros((grid**3, 2), np.float32), + "geometry_valid": np.zeros(2, np.float32), "grid": np.array(grid, np.int32)} + result["occ"][slots] = 1 + for key in ("xyz", "rgb", "scale", "quat", "opacity"): + result[key][slots] = getattr(splats, key)[keep] + return result + + +def mesh_occupancy(path, transform_path, grid): + import trimesh + if not transform_path: + raise ValueError("Every geometry teacher needs an explicit --*-transform JSON matrix, even for identity.") + transform = np.asarray(json.loads(Path(transform_path).read_text()), dtype=float) + if transform.shape != (4, 4) or not np.isfinite(transform).all() or not np.allclose(transform[3], [0,0,0,1]): + raise ValueError("Transform must be a finite affine 4x4 matrix") + mesh = trimesh.load(str(path), force="mesh", process=False) + if not isinstance(mesh, trimesh.Trimesh) or len(mesh.faces) == 0: + raise ValueError("Geometry teacher must contain a triangle mesh") + mesh.apply_transform(transform) + cell_indices(np.asarray(mesh.vertices), grid) # validates the whole object + points, _ = trimesh.sample.sample_surface(mesh, min(200_000, grid**3 * 16), seed=42) + mask = np.zeros(grid**3, np.float32) + mask[np.unique(cell_indices(points, grid))] = 1 + return mask + + +def cache_sample(root, sample_id, image_path, splats, grid=16, source="idol", + coarse=None, coarse_transform=None, fine=None, fine_transform=None, extra=None): + root = Path(root) + if not sample_id or any(c not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_" for c in sample_id): + raise ValueError("Sample ID may contain only letters, digits, - and _") + if grid not in (8, 16, 32): + raise ValueError("Grid must be 8, 16 or 32") + target = splat_targets(splats, grid) + for channel, mesh, transform in [(0, coarse, coarse_transform), (1, fine, fine_transform)]: + if mesh: + target["geometry"][:, channel] = mesh_occupancy(mesh, transform, grid) + target["geometry_valid"][channel] = 1 + row = {"id": sample_id, "image": f"images/{sample_id}.png", "target": f"targets/{sample_id}.npz", + "source": source, "grid": grid, "teachers": {"appearance": source, + "coarse": "InstantMesh" if coarse else None, "fine": "Hi3D" if fine else None}, + "teacher_gaussians": len(splats.xyz), "retained_gaussians": int(target["occ"].sum()), + "geometry_transforms": {"coarse": json.loads(Path(coarse_transform).read_text()) if coarse else None, + "fine": json.loads(Path(fine_transform).read_text()) if fine else None}, + "metadata": extra or {}} + manifest = root / "manifest.jsonl" + rows = load_manifest(manifest) if manifest.exists() else [] + if any(r["id"] == sample_id for r in rows): + raise ValueError(f"Sample {sample_id} already exists; choose a new ID") + if rows and any(r["grid"] != grid for r in rows): + raise ValueError("Do not mix grid resolutions in one dataset") + image = Image.open(image_path) + image.load() + if root.joinpath(row["image"]).exists() or root.joinpath(row["target"]).exists(): + raise ValueError("Sample files already exist; refusing to overwrite") + root.joinpath("images").mkdir(parents=True, exist_ok=True) + root.joinpath("targets").mkdir(parents=True, exist_ok=True) + image.save(root / row["image"]) + row["image_sha256"] = hashlib.sha256((root / row["image"]).read_bytes()).hexdigest() + np.savez_compressed(root / row["target"], **target) + row["target_sha256"] = hashlib.sha256((root / row["target"]).read_bytes()).hexdigest() + # Single-writer dataset. Atomic manifest replacement avoids partial JSON lines. + tmp = manifest.with_suffix(".tmp") + tmp.write_text("".join(json.dumps(r) + "\n" for r in [*rows, row])) + tmp.replace(manifest) + return row + + +def load_manifest(path): + rows = [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()] + if not rows or len({r["id"] for r in rows}) != len(rows): + raise ValueError("Manifest must contain nonempty, unique sample IDs") + return rows + + +def image_tensor(path, size=128): + """Already-framed RGB or RGBA cutout; black background, square letterbox.""" + with Image.open(path) as original: + rgba = ImageOps.exif_transpose(original).convert("RGBA") + bg = Image.new("RGBA", rgba.size, (0, 0, 0, 255)) + bg.alpha_composite(rgba) + image = ImageOps.pad(bg.convert("RGB"), (size, size), method=Image.Resampling.LANCZOS, color="black") + return torch.from_numpy(np.array(image, dtype=np.float32).transpose(2, 0, 1) / 255) + + +class CachedDataset(Dataset): + def __init__(self, manifest, rows=None): + self.root = Path(manifest).resolve().parent + self.rows = load_manifest(manifest) if rows is None else rows + if not self.rows: + raise ValueError("Empty dataset") + self.grid = self.rows[0]["grid"] + for row in self.rows: + if row["grid"] != self.grid: + raise ValueError("Mixed grids") + for key in ("image", "target"): + p = (self.root / row[key]).resolve() + if not p.is_relative_to(self.root) or not p.is_file(): + raise ValueError(f"Invalid dataset {key} path") + digest = hashlib.sha256(p.read_bytes()).hexdigest() + if digest != row[f"{key}_sha256"]: + raise ValueError(f"Dataset {key} hash mismatch for {row['id']}") + + def __len__(self): + return len(self.rows) + + def __getitem__(self, i): + row = self.rows[i] + with np.load(self.root / row["target"], allow_pickle=False) as cached: + target = {k: torch.from_numpy(cached[k].copy()).float() for k in cached.files if k != "grid"} + return image_tensor(self.root / row["image"]), target diff --git a/src/image3d_student/demo.py b/src/image3d_student/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..33dc1630c4db79ddf1e6e8e11ebb3ec50df3fd77 --- /dev/null +++ b/src/image3d_student/demo.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Lucasgitguy-co +from pathlib import Path + +import numpy as np + +from .data import cache_sample +from .splats import Splats, preview + + +def create_demo(root, count=12, grid=8): + """Procedural ellipsoids only. Never presented as outputs of any real teacher.""" + if count < 4: + raise ValueError("Use at least four fixtures") + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + rng = np.random.default_rng(17) + phi, theta = np.meshgrid(np.linspace(.06, np.pi-.06, 14), np.linspace(0, 2*np.pi, 28, endpoint=False)) + unit = np.stack([np.sin(phi)*np.cos(theta), np.cos(phi), np.sin(phi)*np.sin(theta)], axis=-1).reshape(-1, 3) + for i in range(count): + radii = rng.uniform(.17, .36, 3) + xyz = (unit * radii).astype(np.float32) + colour = rng.uniform(.2, .95, 3) + rgb = np.clip(colour[None] * (.8 + .2 * unit[:, 1:2]), 0, 1) + splats = Splats(xyz, rgb, np.full_like(xyz, .035), np.tile([1,0,0,0], (len(xyz),1)), np.full((len(xyz),1), .9)) + image_path = root / f"fixture-{i:03}.png" + preview(splats, size=128).save(image_path) + cache_sample(root, f"fixture-{i:03}", image_path, splats, grid, source="synthetic", + extra={"purpose": "code verification only", "object_id": f"fixture-{i:03}"}) + image_path.unlink() + return root / "manifest.jsonl" diff --git a/src/image3d_student/engine.py b/src/image3d_student/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..84d4ff6a44d5c2fdb8488d3b6f6c29230be56607 --- /dev/null +++ b/src/image3d_student/engine.py @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Lucasgitguy-co +import json +import random +import time +from pathlib import Path + +import numpy as np +import torch +from torch.utils.data import DataLoader + +from .data import CachedDataset, image_tensor, load_manifest +from .model import Student, distillation_loss +from .splats import Splats, write_ply + + +def device_for(name="auto"): + if name == "auto": + name = "mps" if torch.backends.mps.is_available() else "cpu" + if name == "mps" and not torch.backends.mps.is_available(): + raise ValueError("Metal GPU unavailable in this process. Run in a normal macOS Terminal, or use --device cpu.") + if name not in ("mps", "cpu"): + raise ValueError("This prototype supports cpu and mps") + torch.set_num_threads(min(4, torch.get_num_threads())) + return torch.device(name) + + +def split_rows(rows, fraction, seed): + if not 0 <= fraction < 1: + raise ValueError("Validation fraction must lie in [0,1)") + # Keep repeated views/crops of the same object together. Supply object_id in cache metadata. + groups = {} + for row in rows: + group = row.get("metadata", {}).get("object_id", row["id"]) + groups.setdefault(group, []).append(row) + keys = sorted(groups) + random.Random(seed).shuffle(keys) + n = max(1, round(len(keys) * fraction)) if fraction else 0 + if n >= len(keys): + raise ValueError("Need at least two different objects for validation, or explicitly use --validation-fraction 0 for an overfit test") + val_keys = set(keys[:n]) + train = [r for k in keys if k not in val_keys for r in groups[k]] + val = [r for k in keys if k in val_keys for r in groups[k]] + if {r["image_sha256"] for r in train} & {r["image_sha256"] for r in val}: + raise ValueError("Identical images occur across training/validation. Assign the same object-id to related samples.") + return train, val + + +def on_device(target, device): + return {k: v.to(device) for k, v in target.items()} + + +@torch.no_grad() +def evaluate(model, loader, device): + model.eval() + sums = {} + for image, target in loader: + target = on_device(target, device) + pred = model(image.to(device)) + loss, terms = distillation_loss(pred, target, model.grid) + occupied = pred["occ"].sigmoid() > .5 + truth = target["occ"] > .5 + terms = {**terms, "loss": loss, "occupancy_iou": (occupied & truth).sum() / (occupied | truth).sum().clamp_min(1)} + for key, value in terms.items(): + sums[key] = sums.get(key, 0.) + float(value) + return {k: v / len(loader) for k, v in sums.items()} + + +def save_checkpoint(path, model, optimizer, step, metadata): + payload = {"format": 1, "config": model.config, "model": {k: v.detach().cpu() for k, v in model.state_dict().items()}, + "optimizer": optimizer.state_dict(), "step": step, "metadata": metadata} + tmp = Path(path).with_suffix(".tmp") + torch.save(payload, tmp) + tmp.replace(path) + + +def train(args): + if args.steps < 1 or args.batch_size < 1 or args.lr <= 0 or args.eval_every < 1: + raise ValueError("Steps, batch size, learning rate, and evaluation interval must be positive") + device = device_for(args.device) + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + rows = load_manifest(args.manifest) + synthetic = any(r["source"] == "synthetic" for r in rows) + if synthetic and not args.allow_synthetic: + raise ValueError("Synthetic fixtures are code tests, not teacher data. Add --allow-synthetic for an explicit smoke test.") + train_rows, val_rows = split_rows(rows, args.validation_fraction, args.seed) + train_data = CachedDataset(args.manifest, train_rows) + generator = torch.Generator().manual_seed(args.seed) + loader = DataLoader(train_data, batch_size=args.batch_size, shuffle=True, num_workers=0, generator=generator) + val_loader = DataLoader(CachedDataset(args.manifest, val_rows), batch_size=1) if val_rows else None + model = Student(train_data.grid, args.width).to(device) + optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=1e-4) + run = Path(args.out) + if run.exists() and any(run.iterdir()): + raise ValueError("Run directory must be empty; choose a new --out directory") + run.mkdir(parents=True, exist_ok=True) + meta = {"synthetic_only": all(r["source"] == "synthetic" for r in rows), + "contains_synthetic": synthetic, "device": str(device), "torch": str(torch.__version__), + "seed": args.seed, "train_ids": [r["id"] for r in train_rows], "val_ids": [r["id"] for r in val_rows], + "parameters": sum(p.numel() for p in model.parameters()), + "teachers": sorted({v for r in rows for v in r["teachers"].values() if v}), + "validation": "held-out objects" if val_rows else "none; training overfit test only", + "dataset": str(Path(args.manifest).resolve())} + (run / "run.json").write_text(json.dumps(meta, indent=2)) + print(json.dumps(meta), flush=True) + step, best, started = 0, float("inf"), time.monotonic() + iterator = iter(loader) + try: + while step < args.steps: + try: + images, target = next(iterator) + except StopIteration: + iterator = iter(loader) + images, target = next(iterator) + model.train() + optimizer.zero_grad(set_to_none=True) + pred = model(images.to(device)) + loss, terms = distillation_loss(pred, on_device(target, device), model.grid) + if not torch.isfinite(loss): + raise RuntimeError("Nonfinite training loss; stopping") + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.) + optimizer.step() + step += 1 + if step == 1 or step % args.eval_every == 0 or step == args.steps: + record = {"step": step, "seconds": round(time.monotonic()-started, 3), + "train_loss": float(loss.detach()), **{k: float(v.detach()) for k, v in terms.items()}} + if val_loader: + record["validation"] = evaluate(model, val_loader, device) + score = record["validation"]["loss"] + if score < best: + best = score + save_checkpoint(run / "best.pt", model, optimizer, step, meta) + with (run / "metrics.jsonl").open("a") as handle: + handle.write(json.dumps(record) + "\n") + print(json.dumps(record), flush=True) + save_checkpoint(run / "last.pt", model, optimizer, step, meta) + except KeyboardInterrupt: + print("Interrupted; saving completed optimizer steps.", flush=True) + finally: + if step: + save_checkpoint(run / "last.pt", model, optimizer, step, meta) + return meta + + +@torch.no_grad() +def generate(checkpoint, image, out, device="auto", threshold=.5, max_splats=32768): + if not 0 < threshold < 1 or max_splats < 1: + raise ValueError("Threshold must be in (0,1) and max-splats positive") + device = device_for(device) + state = torch.load(checkpoint, map_location="cpu", weights_only=True) + if state.get("format") != 1 or state.get("step", 0) < 1: + raise ValueError("Expected a trained image3d-student checkpoint") + model = Student(**state["config"]) + model.load_state_dict(state["model"]) + model.to(device).eval() + if device.type == "mps": + torch.mps.synchronize() + started = time.monotonic() + pred = model(image_tensor(image)[None].to(device)) + score = pred["occ"][0, :, 0].sigmoid() + valid = torch.nonzero(score >= threshold).flatten() + if len(valid) == 0: + raise ValueError("No occupied cells above threshold. Train longer or inspect validation; no fabricated fallback splats were exported.") + valid = valid[torch.argsort(score[valid], descending=True)[:max_splats]] + values = {k: pred[k][0, valid].cpu().numpy() for k in ("xyz", "rgb", "scale", "quat", "opacity")} + elapsed = time.monotonic() - started + splats = Splats(**values) + out = Path(out) + if out.exists() or out.with_suffix(".json").exists(): + raise ValueError("Output already exists; choose a new filename") + write_ply(out, splats) + report = {"gaussians": len(valid), "seconds_preprocess_and_forward": elapsed, + "device": str(device), "checkpoint_step": state["step"], "training": state["metadata"], + "note": ("Synthetic-trained output is a pipeline demonstration, not a pretrained image-to-3D result." + if state["metadata"].get("contains_synthetic") else "Experimental cached-target student; inspect unseen-object quality.")} + out.with_suffix(".json").write_text(json.dumps(report, indent=2)) + return report diff --git a/src/image3d_student/model.py b/src/image3d_student/model.py new file mode 100644 index 0000000000000000000000000000000000000000..0284bde54f42a2aeba7d63aa401a52e993d02cff --- /dev/null +++ b/src/image3d_student/model.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Lucasgitguy-co +import math + +import torch +from torch import nn +from torch.nn import functional as F + +from .data import grid_centers + + +class Student(nn.Module): + """Small image-conditioned coordinate MLP; fixed cells avoid unordered point regression. + + 15 Gaussian outputs + 2 auxiliary geometry logits. This is a coarse baseline, + not an architecture expected to reproduce million-splat teacher detail. + """ + def __init__(self, grid=16, width=128): + super().__init__() + if grid not in (8, 16, 32) or width < 16: + raise ValueError("Unsupported grid or width") + self.config = {"grid": grid, "width": width} + self.grid = grid + blocks, channels = [], 3 + for out in (24, 48, 96, 128): + blocks.extend([nn.Conv2d(channels, out, 3, stride=2, padding=1), nn.GroupNorm(8, out), nn.SiLU()]) + channels = out + self.encoder = nn.Sequential(*blocks, nn.AdaptiveAvgPool2d(4), nn.Flatten(), + nn.Linear(128*4*4, width), nn.SiLU()) + centers = torch.from_numpy(grid_centers(grid)) + freq = 2**torch.arange(4, dtype=torch.float32) * math.pi * 2 + angles = centers[..., None] * freq + self.register_buffer("centers", centers) + self.register_buffer("coords", torch.cat([centers, angles.sin().flatten(1), angles.cos().flatten(1)], dim=1)) + self.query = nn.Sequential(nn.Linear(27, width), nn.SiLU()) + self.decoder = nn.Sequential(nn.Linear(width, width), nn.SiLU(), nn.Linear(width, width), nn.SiLU(), nn.Linear(width, 17)) + with torch.no_grad(): + self.decoder[-1].bias.zero_() + self.decoder[-1].bias[0] = -2 + + def forward(self, images): + features = self.encoder(images) + raw = self.decoder(features[:, None, :] + self.query(self.coords)[None]) + quat = raw[..., 10:14] + raw.new_tensor([1, 0, 0, 0]) + return {"occ": raw[..., :1], "xyz": self.centers[None] + raw[..., 1:4].tanh() / (2*self.grid), + "rgb": raw[..., 4:7].sigmoid(), + "scale": (raw[..., 7:10].clamp(-8, 5) - math.log(self.grid*3)).exp(), + "quat": F.normalize(quat, dim=-1, eps=1e-8), "opacity": raw[..., 14:15].sigmoid(), + "geometry": raw[..., 15:17]} + + +def balanced_bce(logits, target): + error = F.binary_cross_entropy_with_logits(logits, target, reduction="none") + positive = (error * target).sum() / target.sum().clamp_min(1) + negative = (error * (1-target)).sum() / (1-target).sum().clamp_min(1) + return (positive + negative) / 2 + + +def distillation_loss(pred, target, grid): + mask = target["occ"] + def masked(value): + return (value * mask).sum() / (mask.sum().clamp_min(1) * value.shape[-1]) + terms = {"occupancy": balanced_bce(pred["occ"], mask), + "position": masked(((pred["xyz"] - target["xyz"]) * grid).square()), + "colour": masked((pred["rgb"] - target["rgb"].clamp(0, 1)).square()), + "scale": masked((pred["scale"].log() - target["scale"].clamp_min(1e-8).log()).square()), + "rotation": masked(1 - (pred["quat"] * target["quat"]).sum(-1, keepdim=True).square().clamp(0, 1)), + "opacity": masked((pred["opacity"] - target["opacity"]).square())} + geometry = pred["geometry"].sum() * 0 + for i in range(2): + available = target["geometry_valid"][:, i] > 0 + if available.any(): + geometry = geometry + balanced_bce(pred["geometry"][available, :, i:i+1], target["geometry"][available, :, i:i+1]) + terms["geometry"] = geometry + total = terms["occupancy"] + terms["position"] + 2*terms["colour"] + .1*terms["scale"] + .1*terms["rotation"] + terms["opacity"] + .25*geometry + return total, terms diff --git a/src/image3d_student/splats.py b/src/image3d_student/splats.py new file mode 100644 index 0000000000000000000000000000000000000000..3f0e197bd80a1669f566a909b0198e901776c077 --- /dev/null +++ b/src/image3d_student/splats.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Lucasgitguy-co +"""Activated Gaussian attributes and standard degree-zero 3DGS PLY IO. + +Coordinates are preserved as supplied. Idol's *exported PLY* uses its y-up frame; +we do not apply Idol's internal z-up conversion a second time. +""" +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +from plyfile import PlyData, PlyElement + +C0 = 0.28209479177387814 + + +@dataclass +class Splats: + xyz: np.ndarray + rgb: np.ndarray + scale: np.ndarray + quat: np.ndarray # w,x,y,z + opacity: np.ndarray + + def __post_init__(self): + n = len(self.xyz) + if n == 0: + raise ValueError("No Gaussians: refusing to write or train on an empty object.") + for name, width in [("xyz", 3), ("rgb", 3), ("scale", 3), ("quat", 4), ("opacity", 1)]: + a = np.asarray(getattr(self, name), dtype=np.float32).reshape(-1, width) + if a.shape != (n, width) or not np.isfinite(a).all(): + raise ValueError(f"Invalid {name} shape or nonfinite values") + setattr(self, name, a) + if (self.scale <= 0).any(): + raise ValueError("Gaussian scales must be positive") + if (self.opacity < 0).any() or (self.opacity > 1).any(): + raise ValueError("Opacity must lie in [0,1]") + norm = np.linalg.norm(self.quat, axis=1, keepdims=True) + if (norm < 1e-8).any(): + raise ValueError("Zero quaternion") + self.quat = self.quat / norm + + def take(self, indices): + return Splats(**{k: getattr(self, k)[indices] for k in self.__dataclass_fields__}) + + +def read_ply(path): + v = PlyData.read(str(path))["vertex"].data + required = ["x", "y", "z", "f_dc_0", "f_dc_1", "f_dc_2", "opacity", + "scale_0", "scale_1", "scale_2", "rot_0", "rot_1", "rot_2", "rot_3"] + if not set(required).issubset(v.dtype.names): + raise ValueError("Expected a Gaussian-splat PLY, not a mesh or ordinary point-cloud PLY") + col = lambda names: np.column_stack([v[k] for k in names]).astype(np.float32) + return Splats(col(["x", "y", "z"]), 0.5 + C0 * col([f"f_dc_{i}" for i in range(3)]), + np.exp(np.clip(col([f"scale_{i}" for i in range(3)]), -20, 5)), + col([f"rot_{i}" for i in range(4)]), + 1 / (1 + np.exp(-np.clip(col(["opacity"]), -30, 30)))) + + +def write_ply(path, splats): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + names = ["x", "y", "z", "nx", "ny", "nz", "f_dc_0", "f_dc_1", "f_dc_2", "opacity", + "scale_0", "scale_1", "scale_2", "rot_0", "rot_1", "rot_2", "rot_3"] + opacity = np.clip(splats.opacity, 1e-6, 1 - 1e-6) + values = np.concatenate([splats.xyz, np.zeros_like(splats.xyz), (splats.rgb - 0.5) / C0, + np.log(opacity / (1 - opacity)), np.log(splats.scale), splats.quat], axis=1) + table = np.empty(len(values), dtype=[(k, "f4") for k in names]) + for i, name in enumerate(names): + table[name] = values[:, i] + PlyData([PlyElement.describe(table, "vertex")], text=False).write(str(path)) + + +def rotation_matrices(q): + w, x, y, z = q.T + return np.stack([1-2*(y*y+z*z), 2*(x*y-z*w), 2*(x*z+y*w), + 2*(x*y+z*w), 1-2*(x*x+z*z), 2*(y*z-x*w), + 2*(x*z-y*w), 2*(y*z+x*w), 1-2*(x*x+y*y)], axis=-1).reshape(-1, 3, 3) + + +def preview(splats, size=256, yaw=0.0): + """CPU orthographic Gaussian preview; debugging only, not an evaluation renderer.""" + from PIL import Image + c, s = np.cos(yaw), np.sin(yaw) + view = np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]], dtype=np.float32) + xyz = splats.xyz @ view.T + rot = rotation_matrices(splats.quat) + cov = (rot * (splats.scale ** 2)[:, None, :]) @ rot.transpose(0, 2, 1) + cov = view[None] @ cov @ view.T[None] + pixels = size / 1.25 + canvas = np.zeros((size, size, 3), dtype=np.float32) + # Back to front, camera on +z. Image y points down. + for i in np.argsort(xyz[:, 2]): + center = np.array([xyz[i, 0], -xyz[i, 1]]) * pixels + size / 2 + flip = np.diag([1., -1.]) + covariance = flip @ cov[i, :2, :2] @ flip * pixels ** 2 + np.eye(2) * 0.3 + radius = min(size, int(np.ceil(3 * np.sqrt(np.linalg.eigvalsh(covariance).max())))) + lo = np.maximum(0, np.floor(center - radius).astype(int)) + hi = np.minimum(size, np.ceil(center + radius + 1).astype(int)) + if np.any(hi <= lo): + continue + yy, xx = np.mgrid[lo[1]:hi[1], lo[0]:hi[0]] + delta = np.stack([xx - center[0], yy - center[1]], axis=-1) + exponent = np.einsum("...i,ij,...j->...", delta, np.linalg.inv(covariance), delta) + alpha = (splats.opacity[i, 0] * np.exp(-0.5 * exponent)).clip(0, .99)[..., None] + patch = canvas[lo[1]:hi[1], lo[0]:hi[0]] + patch[:] = patch * (1-alpha) + np.clip(splats.rgb[i], 0, 1) * alpha + return Image.fromarray((canvas.clip(0, 1) * 255).astype(np.uint8)) diff --git a/src/image3d_student/teacher_idol.py b/src/image3d_student/teacher_idol.py new file mode 100644 index 0000000000000000000000000000000000000000..0eceaa032171efaff7d01acb9eb4c0176e58d011 --- /dev/null +++ b/src/image3d_student/teacher_idol.py @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Lucasgitguy-co +"""Experimental, staged Idol inference for Apple MPS. + +Upstream revision: 4702a7f00850e34b6e83a2170cc123c3e9570744. +No weights are bundled. Full-weight inference has NOT been verified by this project. +""" +import gc +import json +import os +import shutil +from pathlib import Path + +import numpy as np +import torch +from PIL import Image + +from .engine import device_for + + +def patch_attention(layers, query_chunk=128): + """Bound attention working memory by splitting only the query dimension. + + Each query still sees ALL keys/values. Does not split or change softmax's key axis. + Disable CUDA-specific optimized sampling and avoid complex RoPE tensors on MPS. + """ + if query_chunk < 1: + raise ValueError("query_chunk must be positive") + from torch.nn import functional as F + + def attention(*args): + q, k, v = args[0].unbind(dim=2) if len(args) == 1 else args + q, k, v = (t.permute(0, 2, 1, 3) for t in (q, k, v)) + chunks = [F.scaled_dot_product_attention(q[:, :, start:start+query_chunk], k, v) + for start in range(0, q.shape[2], query_chunk)] + return torch.cat(chunks, dim=2).permute(0, 2, 1, 3) + + layers.scaled_dot_product_attention = attention + layers.RePo3DRotaryEmbedding.real_valued = True + return attention + + +def release(device): + gc.collect() + if device.type == "mps": + torch.mps.synchronize() + torch.mps.empty_cache() + + +@torch.no_grad() +def run_idol(args): + if args.steps < 1 or args.gaussians < 32768 or args.gaussians > 1048576 or args.gaussians % 32: + raise ValueError("Steps must be positive; Idol requires 32768..1048576 Gaussians, divisible by 32") + device = device_for(args.device) + cache = Path(args.cache).resolve() + cache.mkdir(parents=True, exist_ok=True) + os.environ["HF_HOME"] = str(cache) + os.environ["HF_HUB_OFFLINE"] = "0" if args.allow_download else "1" + os.environ["ATTN_BACKEND"] = "sdpa" + if args.allow_download and shutil.disk_usage(cache).free < 16 * 1024**3: + raise ValueError("Less than 16 GiB disk space available. This conservative download guard leaves room for teacher weights, encoders and temporary files. Free space, or use an existing complete cache without --allow-download.") + try: + from huggingface_hub import hf_hub_download, snapshot_download + from safetensors.torch import load_file + from tensordict import TensorDict + from idol.conditioning import ImageEncoder, require_encoder_access + from idol.models import BiRefNet, GaussianFixedlenDecoder, LatentSeqMMFlowModel, OctreeProbabilityFixedlenDecoder, layers + from idol.preprocess import CROP_MARGIN, preprocess_image, has_real_alpha + from idol.sampler import sample_flow + from idol.camera import decode_camera + except ImportError as exc: + raise RuntimeError("Install the optional teacher dependencies: python -m pip install '.[teacher]'") from exc + out = Path(args.out) + if out.exists(): + raise ValueError("Teacher output directory already exists; choose a new --out") + # Local-only by default. Hugging Face authentication is never written to project files. + config_path = hf_hub_download(args.weights, "config.json", local_files_only=not args.allow_download) + config = json.loads(Path(config_path).read_text()) + encoders = config["image_encoders"] + require_encoder_access([encoders["dinov3"], encoders["flux2_vae"]]) + weights = Path(snapshot_download(args.weights, allow_patterns=["config.json", "*.safetensors"], + local_files_only=not args.allow_download)) + patch_attention(layers, args.query_chunk) + torch.manual_seed(args.seed) + + def load(model, name): + state = load_file(str(weights / f"{name}.safetensors"), device="cpu") + model.load_state_dict(state, strict=True) + del state + return model.eval().requires_grad_(False).to(device) + + print("Experimental Idol MPS adapter: full-weight compatibility and quality remain unverified.", flush=True) + with Image.open(args.image) as opened: + image = opened.copy() + background = None + if not has_real_alpha(image): + print("Stage 1: background removal", flush=True) + background = load(BiRefNet(), "birefnet") + prepared = preprocess_image(image, background, config["image_size"], CROP_MARGIN[args.mode]) + del background + release(device) + pixels = torch.from_numpy(np.array(prepared)).permute(2, 0, 1).float()[None].div(255).to(device) + print("Stage 2: image conditioning", flush=True) + encoder = ImageEncoder(encoders["dinov3"], encoders["flux2_vae"], device="cpu") + # Both encoders initially reside on CPU; only the active encoder is moved to MPS. + encoder.dinov3.to(device) + feature1 = encoder._dinov3_tokens(pixels) + del encoder.dinov3 + release(device) + encoder.flux2_vae.to(device) + feature2 = encoder._flux2_tokens(pixels) + del encoder, pixels + release(device) + cond = TensorDict({"feature1": feature1, "feature2": feature2}, batch_size=1, device=device) + print("Stage 3: latent sampling", flush=True) + model = load(LatentSeqMMFlowModel(**config["denoiser"]["args"]), "denoiser") + noise = TensorDict({"latent": torch.randn(1, model.q_token_length, model.in_channels, device=device), + "camera": torch.randn(1, 1, model.cam_channels, device=device)}, batch_size=1, device=device) + result = sample_flow(model, noise, cond=cond, neg_cond=cond.apply(torch.zeros_like), steps=args.steps, + rescale_t=3., cfg_strength=3., verbose=True, batch_cfg=False) + latent, camera = result["latent"], decode_camera(result["camera"][0, 0]) + del model, noise, cond, feature1, feature2, result + release(device) + print("Stage 4: octree decoding", flush=True) + model = load(OctreeProbabilityFixedlenDecoder(**config["decoder"]["args"]), "decoder") + per_anchor = config["decoder_gs"]["args"]["representation_config"]["num_gaussians"] + anchors = OctreeProbabilityFixedlenDecoder.sample(model, latent, num_points=args.gaussians // per_anchor, + level=config["max_voxel_level"]) + del model + release(device) + print("Stage 5: Gaussian decoding", flush=True) + model = load(GaussianFixedlenDecoder(**config["decoder_gs"]["args"]), "decoder_gs") + gaussian = model.to_representation(anchors, model(anchors, latent))[0] + out.mkdir(parents=True) + gaussian.save_ply(str(out / "teacher.ply")) + prepared.save(out / "input.png") + (out / "metadata.json").write_text(json.dumps({"teacher": args.weights, "steps": args.steps, + "gaussians": args.gaussians, "seed": args.seed, "device": str(device), "camera": camera, + "sdk_revision": "4702a7f00850e34b6e83a2170cc123c3e9570744", "weights_snapshot": weights.name, + "coordinate_frame": "Idol exported PLY (y-up)", "adapter": "experimental staged MPS"}, indent=2)) + print(f"Teacher output saved to {out}", flush=True) diff --git a/tests/test_mps.py b/tests/test_mps.py new file mode 100644 index 0000000000000000000000000000000000000000..dc5d24a564aaeb881834ca3dd2ea5b0855839988 --- /dev/null +++ b/tests/test_mps.py @@ -0,0 +1,47 @@ +"""Real Metal tests; teacher uses random tiny modules, never pretrained weights.""" +import pytest +import torch + +from image3d_student.model import Student +from image3d_student.teacher_idol import patch_attention + +pytestmark = pytest.mark.skipif(not torch.backends.mps.is_available(), reason="Metal GPU not visible") + + +def test_default_student_backward_on_metal(): + model = Student().to("mps") + result = model(torch.rand(1,3,128,128,device="mps")) + loss = sum(t.square().mean() for t in result.values()) + loss.backward() + assert torch.isfinite(loss) + assert result["xyz"].shape == (1,4096,3) + assert all(p.grad is None or torch.isfinite(p.grad).all() for p in model.parameters()) + + +@torch.no_grad() +def test_tiny_real_idol_architecture_on_metal(): + pytest.importorskip("idol") + from tensordict import TensorDict + from idol.models import LatentSeqMMFlowModel, OctreeProbabilityFixedlenDecoder, GaussianFixedlenDecoder, layers + from idol.sampler import sample_flow + patch_attention(layers, query_chunk=4) + model = LatentSeqMMFlowModel(q_token_length=8,in_channels=4,cam_channels=6, + model_channels=32,cond_channels=12,cond2_channels=8,num_blocks=1, + num_refiner_blocks=2,num_heads=2,num_head_channels=16).eval().to("mps") + noise = TensorDict({"latent": torch.randn(1,8,4,device="mps"), + "camera": torch.randn(1,1,6,device="mps")}, batch_size=1, device="mps") + cond = TensorDict({"feature1": torch.randn(1,6,12,device="mps"), + "feature2": torch.randn(1,6,8,device="mps")}, batch_size=1, device="mps") + result = sample_flow(model,noise,cond,cond.apply(torch.zeros_like),steps=2,verbose=False,batch_cfg=False) + latent = result["latent"] + assert torch.isfinite(latent).all() + octree = OctreeProbabilityFixedlenDecoder(32,4,1,2).eval().to("mps") + points = octree.sample(octree,latent,num_points=8,level=3) + assert points.shape == (1,8,3) + config = {"num_gaussians": 4, "lr": {k: 1. for k in ["_xyz","_features_dc","_scaling","_rotation","_opacity"]}, + "perturbe_size": .1, "scaling_bias": .01, "opacity_bias": .8, "3d_filter_kernel_size": .001} + decoder = GaussianFixedlenDecoder(3,32,4,1,2,config).eval().to("mps") + gaussian = decoder.to_representation(points,decoder(points,latent))[0] + assert gaussian.xyz.shape == (32,3) + for value in (gaussian.xyz,gaussian.features_dc,gaussian.scaling,gaussian.rotation,gaussian.opacity): + assert torch.isfinite(value).all() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..d196519740db1c54b433444aac9d8765f506044b --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,142 @@ +import json +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from PIL import Image + +from image3d_student.data import CachedDataset, cache_sample, cell_indices, splat_targets +from image3d_student.engine import split_rows +from image3d_student.model import Student, distillation_loss +from image3d_student.splats import Splats, read_ply, write_ply +from image3d_student.teacher_idol import patch_attention + + +@pytest.fixture +def splats(): + return Splats(np.array([[-.2, .1, .0], [.2, -.1, .3]]), + np.array([[.9,.1,.2], [.1,.8,.4]]), np.array([[.01,.02,.03],[.04,.01,.02]]), + np.array([[1,0,0,0], [.5,.5,.5,.5]]), np.array([[.8],[.9]])) + + +def test_ply_roundtrip_preserves_frame_and_attributes(tmp_path, splats): + path = tmp_path / "splats.ply" + write_ply(path, splats) + reread = read_ply(path) + for name in splats.__dataclass_fields__: + np.testing.assert_allclose(getattr(reread, name), getattr(splats, name), atol=1e-6) + + +def test_non_gaussian_ply_rejected(tmp_path): + import trimesh + path = tmp_path / "mesh.ply" + trimesh.creation.box().export(path) + with pytest.raises(ValueError, match="Gaussian-splat"): + read_ply(path) + + +def test_coordinates_not_silently_clipped(): + with pytest.raises(ValueError, match="aligned"): + cell_indices(np.array([[1., 0, 0]]), 8) + assert cell_indices(np.array([[.5,.5,.5]]), 8)[0] == 511 + + +def test_voxel_assignment_selects_highest_opacity(splats): + splats.xyz[1] = splats.xyz[0] + .001 + target = splat_targets(splats, 8) + assert target["occ"].sum() == 1 + np.testing.assert_allclose(target["rgb"][target["occ"][:,0] > 0][0], splats.rgb[1]) + + +def test_cache_integrity_and_duplicate_protection(tmp_path, splats): + image = tmp_path / "input.png" + Image.new("RGB", (32,32), "red").save(image) + root = tmp_path / "dataset" + cache_sample(root, "a", image, splats, 8) + dataset = CachedDataset(root / "manifest.jsonl") + tensor, target = dataset[0] + assert tensor.shape == (3,128,128) + assert target["occ"].sum() == 2 + with pytest.raises(ValueError, match="already exists"): + cache_sample(root, "a", image, splats, 8) + (root / "images/a.png").write_bytes(b"changed") + with pytest.raises(ValueError, match="hash mismatch"): + CachedDataset(root / "manifest.jsonl") + + +def test_geometry_import_requires_frame_and_marks_teachers(tmp_path, splats): + import trimesh + image = tmp_path / "input.png" + Image.new("RGB", (32,32), "green").save(image) + mesh = tmp_path / "shape.obj" + trimesh.creation.icosphere(subdivisions=1, radius=.3).export(mesh) + with pytest.raises(ValueError, match="explicit"): + cache_sample(tmp_path / "bad", "a", image, splats, 8, coarse=mesh) + transform = tmp_path / "identity.json" + transform.write_text(json.dumps(np.eye(4).tolist())) + root = tmp_path / "good" + row = cache_sample(root, "a", image, splats, 8, coarse=mesh, coarse_transform=transform, + fine=mesh, fine_transform=transform) + assert row["teachers"] == {"appearance": "idol", "coarse": "InstantMesh", "fine": "Hi3D"} + with np.load(root / row["target"]) as target: + assert target["geometry_valid"].tolist() == [1,1] + assert target["geometry"].sum() > 0 + + +def test_missing_teachers_are_masked_and_quaternions_sign_invariant(splats): + torch.manual_seed(1) + model = Student(8,32) + target = {k: torch.from_numpy(v).float()[None] for k,v in splat_targets(splats,8).items() if k != "grid"} + prediction = model(torch.zeros(1,3,128,128)) + loss1, parts1 = distillation_loss(prediction, target, 8) + target["geometry"][:] = 1 + target["quat"] *= -1 + loss2, parts2 = distillation_loss(prediction, target, 8) + torch.testing.assert_close(loss1, loss2) + assert parts1["geometry"] == 0 + assert parts2["geometry"] == 0 + target["geometry_valid"][:] = 1 + loss3, parts3 = distillation_loss(prediction, target, 8) + assert parts3["geometry"] > 0 + assert loss3 > loss2 + + +def test_query_chunk_attention_matches_full_attention(): + from torch.nn import functional as F + torch.manual_seed(7) + layers = SimpleNamespace(RePo3DRotaryEmbedding=type("RoPE", (), {})) + chunked = patch_attention(layers, 3) + q = torch.randn(2,11,2,8) + k = torch.randn(2,17,2,8) + v = torch.randn(2,17,2,8) + expected = F.scaled_dot_product_attention(q.permute(0,2,1,3), k.permute(0,2,1,3), v.permute(0,2,1,3)).permute(0,2,1,3) + torch.testing.assert_close(chunked(q,k,v), expected, atol=1e-6, rtol=1e-5) + qkv = torch.randn(1,7,3,2,8) + q,k,v = qkv.unbind(2) + torch.testing.assert_close(chunked(qkv), chunked(q,k,v)) + assert layers.RePo3DRotaryEmbedding.real_valued + + +def test_split_keeps_object_views_together(): + rows = [{"id": str(i), "image_sha256": str(i), "metadata": {"object_id": str(i//2)}} for i in range(8)] + train, val = split_rows(rows, .25, 42) + assert not {r["metadata"]["object_id"] for r in train} & {r["metadata"]["object_id"] for r in val} + assert len(train) == 6 and len(val) == 2 + + +def test_actual_training_reduces_distillation_loss(splats): + torch.set_num_threads(2) + torch.manual_seed(42) + model = Student(8,32) + target = {k: torch.from_numpy(v).float()[None] for k,v in splat_targets(splats,8).items() if k != "grid"} + image = torch.rand(1,3,128,128) + optimizer = torch.optim.Adam(model.parameters(), lr=.003) + initial = float(distillation_loss(model(image), target, 8)[0].detach()) + for _ in range(25): + optimizer.zero_grad() + loss, _ = distillation_loss(model(image), target, 8) + loss.backward() + optimizer.step() + final = float(distillation_loss(model(image), target, 8)[0].detach()) + assert final < initial * .65, (initial, final)