Spaces:
Running
Running
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +2 -0
- trackio/CHANGELOG.md +517 -0
- trackio/__init__.py +1280 -0
- trackio/_unsupported.py +48 -0
- trackio/_vendor/__init__.py +0 -0
- trackio/_vendor/gradio_exceptions.py +6 -0
- trackio/_vendor/networking.py +81 -0
- trackio/_vendor/tunneling.py +191 -0
- trackio/agent_sessions.py +533 -0
- trackio/alerts.py +184 -0
- trackio/api.py +149 -0
- trackio/apple_gpu.py +206 -0
- trackio/artifact.py +774 -0
- trackio/asgi_app.py +538 -0
- trackio/assets/badge.png +3 -0
- trackio/assets/trackio_logo_dark.png +0 -0
- trackio/assets/trackio_logo_light.png +0 -0
- trackio/assets/trackio_logo_old.png +3 -0
- trackio/assets/trackio_logo_type_dark.png +0 -0
- trackio/assets/trackio_logo_type_dark_transparent.png +0 -0
- trackio/assets/trackio_logo_type_light.png +0 -0
- trackio/assets/trackio_logo_type_light_transparent.png +0 -0
- trackio/bucket_storage.py +214 -0
- trackio/bump.py +614 -0
- trackio/cas.py +214 -0
- trackio/cli.py +0 -0
- trackio/cli_helpers.py +673 -0
- trackio/commit_scheduler.py +306 -0
- trackio/compression.py +169 -0
- trackio/context_vars.py +21 -0
- trackio/cpu.py +303 -0
- trackio/deploy.py +1332 -0
- trackio/dummy_commit_scheduler.py +19 -0
- trackio/exceptions.py +2 -0
- trackio/fragments.py +389 -0
- trackio/frontend/dist/assets/EXT_lights_area-DpkQYXlQ.js +1 -0
- trackio/frontend/dist/assets/EXT_lights_ies-Qe4ehPsC.js +1 -0
- trackio/frontend/dist/assets/EXT_lights_image_based-CEXtWEZ3.js +1 -0
- trackio/frontend/dist/assets/EXT_mesh_gpu_instancing-B3VxVLJg.js +1 -0
- trackio/frontend/dist/assets/EXT_meshopt_compression-Pa499AYZ.js +1 -0
- trackio/frontend/dist/assets/EXT_texture_avif-Nv7VBjPQ.js +1 -0
- trackio/frontend/dist/assets/EXT_texture_webp-BDYrjNRo.js +1 -0
- trackio/frontend/dist/assets/ExtrasAsMetadata-Cp9P1Lmx.js +1 -0
- trackio/frontend/dist/assets/KHR_animation_pointer-DbEjDTwi.js +1 -0
- trackio/frontend/dist/assets/KHR_draco_mesh_compression-BEx33bus.js +1 -0
- trackio/frontend/dist/assets/KHR_interactivity-K8Kioa6L.js +2 -0
- trackio/frontend/dist/assets/KHR_lights_punctual-CCzRzwty.js +1 -0
- trackio/frontend/dist/assets/KHR_materials_anisotropy-DeuhwPRx.js +1 -0
- trackio/frontend/dist/assets/KHR_materials_clearcoat-CqejipXp.js +1 -0
- trackio/frontend/dist/assets/KHR_materials_coat-DOoUAE9E.js +1 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
trackio/assets/badge.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
trackio/assets/trackio_logo_old.png filter=lfs diff=lfs merge=lfs -text
|
trackio/CHANGELOG.md
ADDED
|
@@ -0,0 +1,517 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# trackio
|
| 2 |
+
|
| 3 |
+
## 0.41.0
|
| 4 |
+
|
| 5 |
+
### Features
|
| 6 |
+
|
| 7 |
+
- [#703](https://github.com/gradio-app/trackio/pull/703) [`6bdb2d1`](https://github.com/gradio-app/trackio/commit/6bdb2d1f857473c191e76ef011ecc0c14fa01926) - Add `trackio bump` to upgrade a Space to the local Trackio version. Thanks @abidlabs!
|
| 8 |
+
|
| 9 |
+
## 0.40.0
|
| 10 |
+
|
| 11 |
+
### Features
|
| 12 |
+
|
| 13 |
+
- [#700](https://github.com/gradio-app/trackio/pull/700) [`9f8215d`](https://github.com/gradio-app/trackio/commit/9f8215dcebf3544338c58265d768d113c70f10b2) - Invalidate the Spaces logs cache on every committed write. Thanks @abidlabs!
|
| 14 |
+
|
| 15 |
+
## 0.39.0
|
| 16 |
+
|
| 17 |
+
### Features
|
| 18 |
+
|
| 19 |
+
- [#697](https://github.com/gradio-app/trackio/pull/697) [`89c9b1b`](https://github.com/gradio-app/trackio/commit/89c9b1b8e644a4ee8e5ae3e86a77d54e8a09360b) - Let an embedding page ask the dashboard for its current view. Thanks @abidlabs!
|
| 20 |
+
- [#693](https://github.com/gradio-app/trackio/pull/693) [`3db5e38`](https://github.com/gradio-app/trackio/commit/3db5e38a9fd731959b96793c7aebba6c3481615e) - Add overwrite option to log_artifact. Thanks @abidlabs!
|
| 21 |
+
- [#696](https://github.com/gradio-app/trackio/pull/696) [`81ae55f`](https://github.com/gradio-app/trackio/commit/81ae55fe1fe94e3c4205f9bd892a85cfc73b0ee2) - Hide empty dashboard tabs on narrow viewports. Thanks @abidlabs!
|
| 22 |
+
|
| 23 |
+
## 0.38.1
|
| 24 |
+
|
| 25 |
+
### Features
|
| 26 |
+
|
| 27 |
+
- [#692](https://github.com/gradio-app/trackio/pull/692) [`217a949`](https://github.com/gradio-app/trackio/commit/217a94957cce97a3822a908560f49c435b5b5117) - Fix CommitOperationAdd import with huggingface_hub 1.32. Thanks @abidlabs!
|
| 28 |
+
|
| 29 |
+
## 0.38.0
|
| 30 |
+
|
| 31 |
+
### Features
|
| 32 |
+
|
| 33 |
+
- [#684](https://github.com/gradio-app/trackio/pull/684) [`10011b0`](https://github.com/gradio-app/trackio/commit/10011b0590e2a7ebd552837b16f253a665c2128d) - Reduce dashboard canvas GPU memory and render churn. Thanks @abidlabs!
|
| 34 |
+
- [#688](https://github.com/gradio-app/trackio/pull/688) [`e332a26`](https://github.com/gradio-app/trackio/commit/e332a262a45facc371443d7f16e4f7d39e5869a3) - Show Trackio version beneath dashboard logo. Thanks @abidlabs!
|
| 35 |
+
|
| 36 |
+
## 0.37.1
|
| 37 |
+
|
| 38 |
+
### Features
|
| 39 |
+
|
| 40 |
+
- [#678](https://github.com/gradio-app/trackio/pull/678) [`df9168e`](https://github.com/gradio-app/trackio/commit/df9168e2d0de2b284413a7e81915e7c1f76e49ae) - Fix dotted metric names in line plots. Thanks @abidlabs!
|
| 41 |
+
- [#685](https://github.com/gradio-app/trackio/pull/685) [`b7b349b`](https://github.com/gradio-app/trackio/commit/b7b349b838dcba093e15bc329717ccbb2e3e8f92) - Fix OAuth token authorization for Space logging. Thanks @abidlabs!
|
| 42 |
+
- [#686](https://github.com/gradio-app/trackio/pull/686) [`ddcd855`](https://github.com/gradio-app/trackio/commit/ddcd8557f6b55a7569700a083bd5ab2253f65e61) - Responsive auto layout for charts, sidebar, and tabs. Thanks @pngwn!
|
| 43 |
+
|
| 44 |
+
### Fixes
|
| 45 |
+
|
| 46 |
+
- [#680](https://github.com/gradio-app/trackio/pull/680) [`c81e537`](https://github.com/gradio-app/trackio/commit/c81e53765ea3e58bc36e3016f98b74d265926196) - decode UTF-8 blobs in `trackio query` output instead of hex-encoding them, so JSON columns like `config` and `metrics` are readable. Thanks @B1aZer!
|
| 47 |
+
|
| 48 |
+
## 0.37.0
|
| 49 |
+
|
| 50 |
+
### Features
|
| 51 |
+
|
| 52 |
+
- [#672](https://github.com/gradio-app/trackio/pull/672) [`784819b`](https://github.com/gradio-app/trackio/commit/784819b610bbd872bf39ed9ef5d62ece2a5966b5) - Add hierarchical execution spans and a trace inspector with per-operation latency, status, token usage, cost, and input/output details. Thanks @abidlabs!
|
| 53 |
+
|
| 54 |
+
## 0.36.0
|
| 55 |
+
|
| 56 |
+
### Features
|
| 57 |
+
|
| 58 |
+
- [#674](https://github.com/gradio-app/trackio/pull/674) [`ac5f06c`](https://github.com/gradio-app/trackio/commit/ac5f06c551a051ac435913f3210472d6971b1ed5) - fix: make 3D object cards behave like image cards and fix Reset camera. Thanks @abidlabs!
|
| 59 |
+
|
| 60 |
+
## 0.35.0
|
| 61 |
+
|
| 62 |
+
### Features
|
| 63 |
+
|
| 64 |
+
- [#645](https://github.com/gradio-app/trackio/pull/645) [`8fe869f`](https://github.com/gradio-app/trackio/commit/8fe869f956515344fef68ee69230b2e192ea7707) - artifact registry storage and publishing API — link artifact versions into typed, cross-project collections with `Run.link_artifact` or `Artifact.link` (and remove links with `Artifact.unlink`), and manage them via `trackio.Registry`. Thanks @adrien-grl!
|
| 65 |
+
- [#651](https://github.com/gradio-app/trackio/pull/651) [`ae43b85`](https://github.com/gradio-app/trackio/commit/ae43b85ca625a056280278148bab6e34a5c520fa) - add artifact lineage graph. Thanks @Saba9!
|
| 66 |
+
- [#667](https://github.com/gradio-app/trackio/pull/667) [`e029b80`](https://github.com/gradio-app/trackio/commit/e029b8068b43dea4cd97647d86642abb0b587a1f) - add CLI commands to create, inspect, publish to, promote, and unlink from artifact registries. Thanks @abidlabs!
|
| 67 |
+
- [#668](https://github.com/gradio-app/trackio/pull/668) [`ac9073f`](https://github.com/gradio-app/trackio/commit/ac9073f283e5d28975246f91879b1e5ed4dbf590) - Add standalone read-only registry dashboard. Thanks @abidlabs!
|
| 68 |
+
- [#606](https://github.com/gradio-app/trackio/pull/606) [`1dc2dd9`](https://github.com/gradio-app/trackio/commit/1dc2dd9cab0e1603522c43935ebbec6b1f5efb67) - feat(UI): Run comparer - compare (potentially nested) run configs side by side. Thanks @adrien-grl!
|
| 69 |
+
|
| 70 |
+
## 0.34.1
|
| 71 |
+
|
| 72 |
+
### Features
|
| 73 |
+
|
| 74 |
+
- [#658](https://github.com/gradio-app/trackio/pull/658) [`ad4373c`](https://github.com/gradio-app/trackio/commit/ad4373c521a84092f8bf7b12d87d6d9c65bc3078) - Fix remote media upload filenames. Thanks @MiloDavis!
|
| 75 |
+
- [#659](https://github.com/gradio-app/trackio/pull/659) [`e9b6e06`](https://github.com/gradio-app/trackio/commit/e9b6e0601ed41d51d9cffc143d21d282d25ca73a) - Fix unstable metric sampling in charts that happens in `trl>=1.7.0`. Thanks @abidlabs!
|
| 76 |
+
|
| 77 |
+
## 0.34.0
|
| 78 |
+
|
| 79 |
+
### Features
|
| 80 |
+
|
| 81 |
+
- [#656](https://github.com/gradio-app/trackio/pull/656) [`bb888ca`](https://github.com/gradio-app/trackio/commit/bb888ca79dcb1e99572ffb62bdd3b0020d77dc07) - Remove beta status from logbooks. Thanks @abidlabs!
|
| 82 |
+
- [#652](https://github.com/gradio-app/trackio/pull/652) [`b1e0731`](https://github.com/gradio-app/trackio/commit/b1e07316962fefce71856cafcee7b38e1f477a52) - Log arbitrary HTML and Plotly/Matplotlib figures via `trackio.Html`. Thanks @greysonlalonde!
|
| 83 |
+
|
| 84 |
+
## 0.33.0
|
| 85 |
+
|
| 86 |
+
### Features
|
| 87 |
+
|
| 88 |
+
- [#647](https://github.com/gradio-app/trackio/pull/647) [`1d25383`](https://github.com/gradio-app/trackio/commit/1d253830e302d4bcd67f3aae5458e6c70c5f5144) - Increase smoothing upper bound to 100. Thanks @abidlabs!
|
| 89 |
+
- [#648](https://github.com/gradio-app/trackio/pull/648) [`6a1063f`](https://github.com/gradio-app/trackio/commit/6a1063f7c834f850390866b80608ca17c8563199) - Deprecate persisting data to HF Datasets in favor of Buckets. Thanks @abidlabs!
|
| 90 |
+
|
| 91 |
+
### Fixes
|
| 92 |
+
|
| 93 |
+
- [#649](https://github.com/gradio-app/trackio/pull/649) [`8506189`](https://github.com/gradio-app/trackio/commit/8506189b69f6c0361e6420ef545abaad4a773cb3) - render logged histograms in the dashboard's Metrics tab. Thanks @abidlabs!
|
| 94 |
+
|
| 95 |
+
## 0.32.2
|
| 96 |
+
|
| 97 |
+
### Features
|
| 98 |
+
|
| 99 |
+
- [#641](https://github.com/gradio-app/trackio/pull/641) [`2d1c01a`](https://github.com/gradio-app/trackio/commit/2d1c01a5ce0307ec5c81660c512bf6f25dd3120a) - Improve logbook layout and public repository views. Thanks @abidlabs!
|
| 100 |
+
- [#643](https://github.com/gradio-app/trackio/pull/643) [`e8292c5`](https://github.com/gradio-app/trackio/commit/e8292c5c26baa0e7e806144fa1bbf5ecfb492f3d) - docs: document the artifact tables in the storage schema page. Thanks @adrien-grl!
|
| 101 |
+
|
| 102 |
+
## 0.32.1
|
| 103 |
+
|
| 104 |
+
### Features
|
| 105 |
+
|
| 106 |
+
- [#639](https://github.com/gradio-app/trackio/pull/639) [`73971e6`](https://github.com/gradio-app/trackio/commit/73971e66a4f1daf5c42341727349e6f82ab4e887) - Fix logbook code overflow, CLI command truncation, Hub link contrast, resource ID validation, and live visibility detection for referenced trace datasets and Buckets. Thanks @abidlabs!
|
| 107 |
+
|
| 108 |
+
## 0.32.0
|
| 109 |
+
|
| 110 |
+
### Features
|
| 111 |
+
|
| 112 |
+
- [#635](https://github.com/gradio-app/trackio/pull/635) [`b4587b0`](https://github.com/gradio-app/trackio/commit/b4587b0ebce18756a0f38e3721ed2880a7307137) - Add Code & Markdown, Traces, and Workspace logbook views with live local refresh, guided empty states, and privacy-aware manual publishing to Agent Traces Datasets and HF Buckets. Thanks @NielsRogge!
|
| 113 |
+
|
| 114 |
+
## 0.31.5
|
| 115 |
+
|
| 116 |
+
### Features
|
| 117 |
+
|
| 118 |
+
- [#630](https://github.com/gradio-app/trackio/pull/630) [`bed508c`](https://github.com/gradio-app/trackio/commit/bed508c04c487686dea3dea70c67b058dd895f96) - Fix automatic dashboard cells to use the active logbook page instead of creating a page named after the Trackio project. Thanks @abidlabs!
|
| 119 |
+
|
| 120 |
+
## 0.31.4
|
| 121 |
+
|
| 122 |
+
### Features
|
| 123 |
+
|
| 124 |
+
- [#628](https://github.com/gradio-app/trackio/pull/628) [`3eeaa99`](https://github.com/gradio-app/trackio/commit/3eeaa99bd05bef69053b9a1b5c7af3abf56c360c) - let `logbook cell figure` embed image files directly. Thanks @abidlabs!/n `trackio logbook cell figure` now accepts an image path via a new `--image`/n flag, and `--html <file>` transparently embeds the file when it points at an/n image. Previously the only way to add a PNG/JPG figure was to hand-encode it/n into an `<img>` data-URI, and passing an image path to `--html` crashed with a/n `UnicodeDecodeError` (the binary file was read as UTF-8 text). Images are/n embedded as responsive base64 data URIs. The Python API `add_figure_cell` gains/n a matching `image=` parameter.
|
| 125 |
+
|
| 126 |
+
## 0.31.3
|
| 127 |
+
|
| 128 |
+
### Features
|
| 129 |
+
|
| 130 |
+
- [#626](https://github.com/gradio-app/trackio/pull/626) [`174aecf`](https://github.com/gradio-app/trackio/commit/174aecff33e6d1cc8d71965a73538c0510abed46) - Fix logbook Markdown escaping and figure hotspot navigation. Thanks @abidlabs!
|
| 131 |
+
|
| 132 |
+
## 0.31.2
|
| 133 |
+
|
| 134 |
+
### Features
|
| 135 |
+
|
| 136 |
+
- [#624](https://github.com/gradio-app/trackio/pull/624) [`a0918ed`](https://github.com/gradio-app/trackio/commit/a0918edc7594bb211f9e6871546b9d36939f0658) - Fix poster hotspot navigation after iframe load. Thanks @abidlabs!
|
| 137 |
+
|
| 138 |
+
## 0.31.1
|
| 139 |
+
|
| 140 |
+
### Features
|
| 141 |
+
|
| 142 |
+
- [#621](https://github.com/gradio-app/trackio/pull/621) [`a40f1a4`](https://github.com/gradio-app/trackio/commit/a40f1a464c51652ecdeab04d999659d71b76047e) - add native fullscreen mode for logbook figures. Thanks @abidlabs!
|
| 143 |
+
- [#623](https://github.com/gradio-app/trackio/pull/623) [`31fd4d7`](https://github.com/gradio-app/trackio/commit/31fd4d7b4004f0c511484f089e6a550da1b4d71e) - let interactive logbook figures navigate to logbook pages. Thanks @abidlabs!
|
| 144 |
+
|
| 145 |
+
## 0.31.0
|
| 146 |
+
|
| 147 |
+
### Features
|
| 148 |
+
|
| 149 |
+
- [#601](https://github.com/gradio-app/trackio/pull/601) [`8c49140`](https://github.com/gradio-app/trackio/commit/8c491406c54685369d43f3a9c4d4a0e99108bc49) - browse artifacts from the UI. Thanks @Saba9!
|
| 150 |
+
- [#602](https://github.com/gradio-app/trackio/pull/602) [`1ae1558`](https://github.com/gradio-app/trackio/commit/1ae155803b5450b181a47767255421499074c70b) - feat: add Artifact.add_reference to reference external objects without copy. Thanks @adrien-grl!
|
| 151 |
+
|
| 152 |
+
## 0.30.4
|
| 153 |
+
|
| 154 |
+
### Features
|
| 155 |
+
|
| 156 |
+
- [#618](https://github.com/gradio-app/trackio/pull/618) [`64c18f6`](https://github.com/gradio-app/trackio/commit/64c18f65bf98d6f54040395a1ca0473347ba6a2b) - Logbook: share button on figures + `logbook pin` CLI. Thanks @abidlabs!
|
| 157 |
+
|
| 158 |
+
## 0.30.3
|
| 159 |
+
|
| 160 |
+
### Features
|
| 161 |
+
|
| 162 |
+
- [#616](https://github.com/gradio-app/trackio/pull/616) [`645498b`](https://github.com/gradio-app/trackio/commit/645498be37bafab28f884a36e80c5bc0eab06cd9) - Logbook single page. Thanks @abidlabs!
|
| 163 |
+
|
| 164 |
+
## 0.30.2
|
| 165 |
+
|
| 166 |
+
### Features
|
| 167 |
+
|
| 168 |
+
- [#614](https://github.com/gradio-app/trackio/pull/614) [`eba80c7`](https://github.com/gradio-app/trackio/commit/eba80c7464af250519a90b8ac3179ff6652b651a) - Render logbooks as single-page documents with section navigation and contextual resources. Thanks @abidlabs!
|
| 169 |
+
- [#612](https://github.com/gradio-app/trackio/pull/612) [`2c1b943`](https://github.com/gradio-app/trackio/commit/2c1b9436b093e9f9335c99c10fe4b6afa622e314) - Warn that logbook open is experimental. Thanks @abidlabs!
|
| 170 |
+
- [#611](https://github.com/gradio-app/trackio/pull/611) [`96b1d94`](https://github.com/gradio-app/trackio/commit/96b1d94def7596aa1302864cbfc57942dd63822b) - Logbook: auto-capture output artifacts and live-embedded dashboard cells. Thanks @abidlabs!
|
| 171 |
+
|
| 172 |
+
## 0.30.1
|
| 173 |
+
|
| 174 |
+
### Features
|
| 175 |
+
|
| 176 |
+
- [#609](https://github.com/gradio-app/trackio/pull/609) [`2ed1295`](https://github.com/gradio-app/trackio/commit/2ed12959c73aa9c04ca1096cf7a8315ce2886ed1) - Serve live logbook preview from open. Thanks @abidlabs!
|
| 177 |
+
|
| 178 |
+
## 0.30.0
|
| 179 |
+
|
| 180 |
+
### Features
|
| 181 |
+
|
| 182 |
+
- [#603](https://github.com/gradio-app/trackio/pull/603) [`a793fd8`](https://github.com/gradio-app/trackio/commit/a793fd88827482134bce295c6228e117de64566a) - Move default artifact downloads under `./.trackio/artifact-downloads/` so materialized files stay out of the project root. Thanks @abidlabs!
|
| 183 |
+
- [#604](https://github.com/gradio-app/trackio/pull/604) [`dd3b8ee`](https://github.com/gradio-app/trackio/commit/dd3b8eeef4e0da8bf0df6238617e20ae81f3e564) - docs: clarify artifact types. Thanks @abidlabs!
|
| 184 |
+
- [#586](https://github.com/gradio-app/trackio/pull/586) [`7db4c3b`](https://github.com/gradio-app/trackio/commit/7db4c3b6de6ff4903daf7dabed8842f212c8b7fd) - artifact tracking API — `log_artifact`, `use_artifact`, and an `Artifact` class for versioned, named file collections with aliases, content-addressed deduplication, and producer/consumer run lineage; works offline and syncs blobs to Hugging Face Spaces/Datasets. Thanks @Saba9!
|
| 185 |
+
- [#597](https://github.com/gradio-app/trackio/pull/597) [`5ba561e`](https://github.com/gradio-app/trackio/commit/5ba561e1b4023abdf63624f4fb396367ed6e9e76) - Add experiment logbooks: shareable static-Space lab notebooks. Thanks @abidlabs!
|
| 186 |
+
|
| 187 |
+
## 0.29.0
|
| 188 |
+
|
| 189 |
+
### Features
|
| 190 |
+
|
| 191 |
+
- [#593](https://github.com/gradio-app/trackio/pull/593) [`ca4722c`](https://github.com/gradio-app/trackio/commit/ca4722c19d2219a3a78b5977149873b9dff73f34) - Add CLI command to list Trackio Spaces. Thanks @abidlabs!
|
| 192 |
+
|
| 193 |
+
## 0.28.1
|
| 194 |
+
|
| 195 |
+
### Features
|
| 196 |
+
|
| 197 |
+
- [#589](https://github.com/gradio-app/trackio/pull/589) [`5e9905d`](https://github.com/gradio-app/trackio/commit/5e9905d7815bb063f2d9bb38aa76177405c5e2b4) - Reduce Metrics page log payloads by requesting scalar-only metrics and add URL x-axis initialization/fallback handling. Thanks @evalstate!
|
| 198 |
+
- [#590](https://github.com/gradio-app/trackio/pull/590) [`56493a6`](https://github.com/gradio-app/trackio/commit/56493a6e9a0114b8655ab44b8c3dec7bae2a5df5) - Use TRACKIO_WRITE_TOKEN for server write token. Thanks @abidlabs!
|
| 199 |
+
|
| 200 |
+
## 0.28.0
|
| 201 |
+
|
| 202 |
+
### Features
|
| 203 |
+
|
| 204 |
+
- [#582](https://github.com/gradio-app/trackio/pull/582) [`d9dabb5`](https://github.com/gradio-app/trackio/commit/d9dabb5bda8147ad7c9203646358bd0880fa6521) - Add JSONL inbox fragments: durable logging when the Space is unreachable and on network filesystems. Thanks @abidlabs!
|
| 205 |
+
|
| 206 |
+
## 0.27.0
|
| 207 |
+
|
| 208 |
+
### Features
|
| 209 |
+
|
| 210 |
+
- [#567](https://github.com/gradio-app/trackio/pull/567) [`8c0ce21`](https://github.com/gradio-app/trackio/commit/8c0ce21ff9746d7c6a250684b1b5f0f3f04e0520) - Add CPU, RAM, disk, network and sensor metrics monitoring for non-Apple devices. Thanks @yannsadowski!
|
| 211 |
+
- [#571](https://github.com/gradio-app/trackio/pull/571) [`57bc7df`](https://github.com/gradio-app/trackio/commit/57bc7df27ecd7b3ff543dbd2bff5c94182bfb69e) - Fix IndexError when importing CSV with no numeric metric columns. Thanks @Ckal!
|
| 212 |
+
- [#577](https://github.com/gradio-app/trackio/pull/577) [`db0e730`](https://github.com/gradio-app/trackio/commit/db0e730463103142688c1c0efef9ff68a52e281d) - Sort and paginate media items. Thanks @abidlabs!
|
| 213 |
+
- [#576](https://github.com/gradio-app/trackio/pull/576) [`6f635e2`](https://github.com/gradio-app/trackio/commit/6f635e22553ce2deaece6a0d6289475ab8683040) - Render plot/metric titles as searchable DOM text. Thanks @abidlabs!
|
| 214 |
+
- [#579](https://github.com/gradio-app/trackio/pull/579) [`cf7759e`](https://github.com/gradio-app/trackio/commit/cf7759efaab73ce1e98a8a2355dc1f65d54bc775) - Merge image viewer with current media page, credit to @catwell. Thanks @abidlabs!
|
| 215 |
+
|
| 216 |
+
## 0.26.0
|
| 217 |
+
|
| 218 |
+
### Features
|
| 219 |
+
|
| 220 |
+
- [#553](https://github.com/gradio-app/trackio/pull/553) [`06011ac`](https://github.com/gradio-app/trackio/commit/06011acc9c73341fd234f9cd8eaf96d5a34ad8ce) - fix: serve static-Space assets at /static/trackio. Thanks @abidlabs!
|
| 221 |
+
- [#563](https://github.com/gradio-app/trackio/pull/563) [`551569c`](https://github.com/gradio-app/trackio/commit/551569c16fb56ec63249ebdc28348d326ccf7126) - Fix Traces UI a bit. Thanks @abidlabs!
|
| 222 |
+
- [#550](https://github.com/gradio-app/trackio/pull/550) [`5690acd`](https://github.com/gradio-app/trackio/commit/5690acda5da303c63ad332451afeab3e9750fd1a) - fix: keep sparse metrics sparse through smoothing. Thanks @abidlabs!
|
| 223 |
+
- [#551](https://github.com/gradio-app/trackio/pull/551) [`0ef7266`](https://github.com/gradio-app/trackio/commit/0ef72660695cf932f3906ddbf33d41d087280a22) - add "group by" dropdown to sidebar. Thanks @Saba9!
|
| 224 |
+
- [#538](https://github.com/gradio-app/trackio/pull/538) [`a15c1a8`](https://github.com/gradio-app/trackio/commit/a15c1a8877c07514e0596630bb7c7299299994a9) - Subdue empty dashboard tabs. Thanks @abidlabs!
|
| 225 |
+
- [#559](https://github.com/gradio-app/trackio/pull/559) [`0b53a41`](https://github.com/gradio-app/trackio/commit/0b53a413909598f92138b6b6395a91c2d5034faf) - Store traces separately from metrics. Thanks @abidlabs!
|
| 226 |
+
- [#556](https://github.com/gradio-app/trackio/pull/556) [`d110001`](https://github.com/gradio-app/trackio/commit/d110001dbd9f6b262dfe41f2b702e3a71aa0cfc9) - fix: keep selected x-axis option in dropdown and dismiss dropdown on re-click. Thanks @Saba9!
|
| 227 |
+
- [#560](https://github.com/gradio-app/trackio/pull/560) [`aee2923`](https://github.com/gradio-app/trackio/commit/aee2923d3ada4f74d62d065c16f1f6a56a295e48) - Paginate Traces tab with step filter. Thanks @abidlabs!
|
| 228 |
+
|
| 229 |
+
### Fixes
|
| 230 |
+
|
| 231 |
+
- [#540](https://github.com/gradio-app/trackio/pull/540) [`0b674ac`](https://github.com/gradio-app/trackio/commit/0b674ac6438738de89bc5b3fb700ccfd8a39368c) - raise default metrics sampling cap from 1500 to 3000 so client-side smoothing on the Metrics tab runs over higher-resolution data. Thanks @edbeeching!
|
| 232 |
+
|
| 233 |
+
## 0.25.1
|
| 234 |
+
|
| 235 |
+
### Features
|
| 236 |
+
|
| 237 |
+
- [#535](https://github.com/gradio-app/trackio/pull/535) [`d7f1b27`](https://github.com/gradio-app/trackio/commit/d7f1b27a98f185d2d97ef54975d5865e0b5243c9) - Avoid HF token leaks in static snapshots. Thanks @abidlabs!
|
| 238 |
+
|
| 239 |
+
## 0.25.0
|
| 240 |
+
|
| 241 |
+
### Features
|
| 242 |
+
|
| 243 |
+
- [#533](https://github.com/gradio-app/trackio/pull/533) [`08bc5eb`](https://github.com/gradio-app/trackio/commit/08bc5eb090525d3ff5f7fa4233c30c42162aa74c) - Fix Windows-only emoji mojibake when uploading Space README. Thanks @tomaarsen!
|
| 244 |
+
- [#518](https://github.com/gradio-app/trackio/pull/518) [`e7ed176`](https://github.com/gradio-app/trackio/commit/e7ed176da53d8b49290fddd890b3d18c0b9b958f) - Traces in Trackio. Thanks @abidlabs!
|
| 245 |
+
- [#531](https://github.com/gradio-app/trackio/pull/531) [`27a50a3`](https://github.com/gradio-app/trackio/commit/27a50a37362020304b774344b4c774ff548985b6) - Add configurable custom frontends for Trackio. Thanks @abidlabs!
|
| 246 |
+
|
| 247 |
+
## 0.24.2
|
| 248 |
+
|
| 249 |
+
### Features
|
| 250 |
+
|
| 251 |
+
- [#527](https://github.com/gradio-app/trackio/pull/527) [`7d1c0b9`](https://github.com/gradio-app/trackio/commit/7d1c0b9c37ce9a9845e6bbe6c083da9d36084caf) - Fix dashboard UX issues: smoothing in share URL, run selection, and run filtering. Thanks @abidlabs!
|
| 252 |
+
- [#526](https://github.com/gradio-app/trackio/pull/526) [`643878a`](https://github.com/gradio-app/trackio/commit/643878a82985fab9e6675f769ff0107cb46e042a) - Add emoji to README and deploy README content. Thanks @qgallouedec!
|
| 253 |
+
- [#529](https://github.com/gradio-app/trackio/pull/529) [`a77972b`](https://github.com/gradio-app/trackio/commit/a77972b68541ebe9e056824e69c9bbca3979ece4) - Remove pydub dependency. Thanks @abidlabs!
|
| 254 |
+
|
| 255 |
+
## 0.24.1
|
| 256 |
+
|
| 257 |
+
### Features
|
| 258 |
+
|
| 259 |
+
- [#524](https://github.com/gradio-app/trackio/pull/524) [`65a6897`](https://github.com/gradio-app/trackio/commit/65a6897561b465fc8f05550562f8da1ba3c99060) - Fix `trackio skills add`. Thanks @abidlabs!
|
| 260 |
+
- [#522](https://github.com/gradio-app/trackio/pull/522) [`05aaca7`](https://github.com/gradio-app/trackio/commit/05aaca7f166bf7667b60b40656d677532a4bdd6e) - relax `starlette` dependency and fix import style. Thanks @abidlabs!
|
| 261 |
+
- [#525](https://github.com/gradio-app/trackio/pull/525) [`32c05c5`](https://github.com/gradio-app/trackio/commit/32c05c5d5e3aa84ca7099a6fa08a9093ccd4b95f) - Restore sidebar share and embed snippets, and fix query parameter regression. Thanks @abidlabs!
|
| 262 |
+
|
| 263 |
+
## 0.24.0
|
| 264 |
+
|
| 265 |
+
### Features
|
| 266 |
+
|
| 267 |
+
- [#502](https://github.com/gradio-app/trackio/pull/502) [`3b397df`](https://github.com/gradio-app/trackio/commit/3b397dfbaff9de137b088f3cad528117e14faab1) - Add docs on SQL & Parquet schema / format, as well as a new CLI command: `trackio query project --project PROJECT --sql SQL_QUERY`. Thanks @abidlabs!
|
| 268 |
+
- [#506](https://github.com/gradio-app/trackio/pull/506) [`498bbc4`](https://github.com/gradio-app/trackio/commit/498bbc47f66cc90cc5776f363d001a5571941c00) - Scope bucket sync to trackio/ subtree to avoid walking the HF cache. Thanks @abidlabs!
|
| 269 |
+
- [#505](https://github.com/gradio-app/trackio/pull/505) [`8e26ab9`](https://github.com/gradio-app/trackio/commit/8e26ab93b5d9caa2f81334f6fff42fb9cefbb232) - Add an `id` field to `Run` which is used internally, allowing users to have multiple runs with the same run name. Thanks @abidlabs!
|
| 270 |
+
- [#517](https://github.com/gradio-app/trackio/pull/517) [`29e1034`](https://github.com/gradio-app/trackio/commit/29e1034b795567ec5ed6d19c5a946915a6498e2a) - Fix static exports, Space bucket handling, and other misc issues. Thanks @abidlabs!
|
| 271 |
+
- [#489](https://github.com/gradio-app/trackio/pull/489) [`1b96db3`](https://github.com/gradio-app/trackio/commit/1b96db39c8fd4326e621ee2336b0fca4f263a18a) - Remove `gradio` dependency in `trackio` -- only `gradio_client` is needed locally anymore. Also lazily import `pandas` and remove it as a dependency. Thanks @abidlabs!
|
| 272 |
+
- [#513](https://github.com/gradio-app/trackio/pull/513) [`d54d290`](https://github.com/gradio-app/trackio/commit/d54d290fcb1bb08358b558a43a962f78abe990ea) - Reduce HF Spaces 429s: polling tuning and batched metric logs API. Thanks @abidlabs!
|
| 273 |
+
- [#516](https://github.com/gradio-app/trackio/pull/516) [`afe2959`](https://github.com/gradio-app/trackio/commit/afe295988928a3ea3ded38bdb5bb05cca85d3c74) - Fix run list order and legend overflow. Thanks @abidlabs!
|
| 274 |
+
- [#515](https://github.com/gradio-app/trackio/pull/515) [`0a242b8`](https://github.com/gradio-app/trackio/commit/0a242b85127b02f532b24c7fd2bb046580cc7641) - Add Gradio-compatible /gradio_api routes on Spaces. Thanks @abidlabs!
|
| 275 |
+
- [#510](https://github.com/gradio-app/trackio/pull/510) [`60bbc86`](https://github.com/gradio-app/trackio/commit/60bbc86b4e7f880de72075e5bf31b093709bb5a4) - Add server_url and TRACKIO_SERVER_URL for self-hosted servers; space_id and TRACKIO_SPACE_ID take precedence when both are set. Thanks @abidlabs!
|
| 276 |
+
- [#509](https://github.com/gradio-app/trackio/pull/509) [`21c099a`](https://github.com/gradio-app/trackio/commit/21c099aa830a278973fab4c7c58a0139f417caa4) - Fix: Open browser with write_token so trackio show allows mutations. Thanks @abidlabs!
|
| 277 |
+
|
| 278 |
+
## 0.23.0
|
| 279 |
+
|
| 280 |
+
### Features
|
| 281 |
+
|
| 282 |
+
- [#494](https://github.com/gradio-app/trackio/pull/494) [`e8a897d`](https://github.com/gradio-app/trackio/commit/e8a897d2266d9b2558f72d768b0b21f4d0a8781b) - Add a settings/CLI page to Trackio. Thanks @abidlabs!
|
| 283 |
+
- [#481](https://github.com/gradio-app/trackio/pull/481) [`882647e`](https://github.com/gradio-app/trackio/commit/882647ec1599cf04500d03b5ca75ddc2733682e2) - Add multi-GPU system metrics support. Thanks @Saba9!
|
| 284 |
+
- [#485](https://github.com/gradio-app/trackio/pull/485) [`46a3cc3`](https://github.com/gradio-app/trackio/commit/46a3cc3758719e171417612efee102a487e71ebd) - Fix/remove flaky E2E space tests. Thanks @abidlabs!
|
| 285 |
+
- [#501](https://github.com/gradio-app/trackio/pull/501) [`06ea885`](https://github.com/gradio-app/trackio/commit/06ea8852f5e40ab3f1cf629a0a01af5c17f847a1) - Fix SQLite corruption on bucket-mounted Spaces. Thanks @abidlabs!
|
| 286 |
+
- [#496](https://github.com/gradio-app/trackio/pull/496) [`af23d74`](https://github.com/gradio-app/trackio/commit/af23d74438b146c4a3512ace15ea984656e943ed) - Prevent trackio errors from crashing the user's training loop. Thanks @abidlabs!
|
| 287 |
+
|
| 288 |
+
## 0.22.0
|
| 289 |
+
|
| 290 |
+
### Features
|
| 291 |
+
|
| 292 |
+
- [#484](https://github.com/gradio-app/trackio/pull/484) [`cc05ada`](https://github.com/gradio-app/trackio/commit/cc05ada8e89773f3a894af99b801ef680f64418f) - Fix duplicate columns in parquet export. Thanks @abidlabs!
|
| 293 |
+
- [#487](https://github.com/gradio-app/trackio/pull/487) [`853f764`](https://github.com/gradio-app/trackio/commit/853f7646a70d12633afaa4f69db86425aa665413) - Relax `PIL` dependency and remove `plotly` as it's no longer used. Thanks @abidlabs!
|
| 294 |
+
|
| 295 |
+
## 0.21.2
|
| 296 |
+
|
| 297 |
+
### Features
|
| 298 |
+
|
| 299 |
+
- [#482](https://github.com/gradio-app/trackio/pull/482) [`f62180a`](https://github.com/gradio-app/trackio/commit/f62180a0218bc99a259d5ca110a0384a6cae11c8) - Use server-side bucket copy when freezing Spaces. Thanks @abidlabs!
|
| 300 |
+
|
| 301 |
+
## 0.21.1
|
| 302 |
+
|
| 303 |
+
### Features
|
| 304 |
+
|
| 305 |
+
- [#475](https://github.com/gradio-app/trackio/pull/475) [`fcb476c`](https://github.com/gradio-app/trackio/commit/fcb476cd37a40923e9679aaf966f41d582a878a8) - Tweaks. Thanks @abidlabs!
|
| 306 |
+
- [#477](https://github.com/gradio-app/trackio/pull/477) [`7d52dfd`](https://github.com/gradio-app/trackio/commit/7d52dfdce5b6eff6a34501a6d5a620220663cf09) - Fix `.sync()` and add `.freeze()` as a separate methods. Thanks @abidlabs!
|
| 307 |
+
|
| 308 |
+
## 0.21.0
|
| 309 |
+
|
| 310 |
+
### Features
|
| 311 |
+
|
| 312 |
+
- [#467](https://github.com/gradio-app/trackio/pull/467) [`f357deb`](https://github.com/gradio-app/trackio/commit/f357debf78957e4c1f2b901bee4f77cf397298b4) - Allow logged metrics as x-axis choices. Thanks @abidlabs!
|
| 313 |
+
- [#474](https://github.com/gradio-app/trackio/pull/474) [`655673d`](https://github.com/gradio-app/trackio/commit/655673d4c6b7c8b7ee8f87f2589f2dbbc3d2ef91) - Fix file descriptor leak from `sqlite3.connect`. Thanks @abidlabs!
|
| 314 |
+
- [#470](https://github.com/gradio-app/trackio/pull/470) [`bea8c9d`](https://github.com/gradio-app/trackio/commit/bea8c9dcae0b59d071b6c779c97ee525c9bbf6e7) - Restores tooltips to line plots and fixes the call to uses TTL instead of OAuth. Thanks @abidlabs!
|
| 315 |
+
- [#471](https://github.com/gradio-app/trackio/pull/471) [`246fce0`](https://github.com/gradio-app/trackio/commit/246fce0a01619e1c2c538c67b3e460883334d500) - Deprecate dataset backend in favor of buckets. Thanks @abidlabs!
|
| 316 |
+
- [#465](https://github.com/gradio-app/trackio/pull/465) [`3e11174`](https://github.com/gradio-app/trackio/commit/3e1117438bb8168b802245a33059affa558ae519) - Use HF buckets as backend. Thanks @abidlabs!
|
| 317 |
+
- [#469](https://github.com/gradio-app/trackio/pull/469) [`915d170`](https://github.com/gradio-app/trackio/commit/915d17045133172b59195acfdcc70709229668aa) - Make static Spaces work with Buckets and also allow conversion from Gradio SDK to Static Spaces. Thanks @abidlabs!
|
| 318 |
+
|
| 319 |
+
## 0.20.2
|
| 320 |
+
|
| 321 |
+
### Features
|
| 322 |
+
|
| 323 |
+
- [#464](https://github.com/gradio-app/trackio/pull/464) [`c89ebb3`](https://github.com/gradio-app/trackio/commit/c89ebb3b50f695bc7f16cbc6f46dce86f79a01e9) - Improve rendering of curves. Thanks @abidlabs!
|
| 324 |
+
- [#462](https://github.com/gradio-app/trackio/pull/462) [`9160b78`](https://github.com/gradio-app/trackio/commit/9160b78ff6f258f0b87a4f34a24e7d0b5dfbf2fb) - Refactor plot title to display only the metric name without the path. Thanks @qgallouedec!
|
| 325 |
+
|
| 326 |
+
## 0.20.1
|
| 327 |
+
|
| 328 |
+
### Features
|
| 329 |
+
|
| 330 |
+
- [#454](https://github.com/gradio-app/trackio/pull/454) [`22881db`](https://github.com/gradio-app/trackio/commit/22881dbbbb6b81197a00a19853771007093d61e4) - Bar chart single point. Thanks @abidlabs!
|
| 331 |
+
- [#455](https://github.com/gradio-app/trackio/pull/455) [`f8db51a`](https://github.com/gradio-app/trackio/commit/f8db51a20ca61ef703f3f2c2ee1ebd9c4f239cf2) - Adds a static Trackio mode via `trackio.sync(sdk="static")` and support for the `TRACKIO_SPACE_ID` environment variable. Thanks @abidlabs!
|
| 332 |
+
|
| 333 |
+
## 0.20.0
|
| 334 |
+
|
| 335 |
+
### Features
|
| 336 |
+
|
| 337 |
+
- [#450](https://github.com/gradio-app/trackio/pull/450) [`b0571ef`](https://github.com/gradio-app/trackio/commit/b0571ef6207a1ce346696f858ad2b7b584dd194f) - Use Svelte source for Gradio components directly in Trackio dashboard. Thanks @abidlabs!
|
| 338 |
+
|
| 339 |
+
## 0.19.0
|
| 340 |
+
|
| 341 |
+
### Features
|
| 342 |
+
|
| 343 |
+
- [#445](https://github.com/gradio-app/trackio/pull/445) [`cef4a58`](https://github.com/gradio-app/trackio/commit/cef4a583cb76f4091fc6c0e5783124ee84f8e243) - Add remote HF Space support to CLI. Thanks @abidlabs!
|
| 344 |
+
- [#444](https://github.com/gradio-app/trackio/pull/444) [`358f2a9`](https://github.com/gradio-app/trackio/commit/358f2a9ca238ee8b90b5a8c96220da287e0698fb) - Fix alerts placeholder flashing on reports page. Thanks @abidlabs!
|
| 345 |
+
|
| 346 |
+
## 0.18.0
|
| 347 |
+
|
| 348 |
+
### Features
|
| 349 |
+
|
| 350 |
+
- [#435](https://github.com/gradio-app/trackio/pull/435) [`4a47112`](https://github.com/gradio-app/trackio/commit/4a471128e18a39e45fad48a67fd711c5ae9e4aed) - feat: allow hiding section header accordions. Thanks @Saba9!
|
| 351 |
+
- [#439](https://github.com/gradio-app/trackio/pull/439) [`18e9650`](https://github.com/gradio-app/trackio/commit/18e96503d5a3a7cf926e92782d457e23c19942bd) - Add alerts with webhooks, CLI, and documentation. Thanks @abidlabs!
|
| 352 |
+
- [#438](https://github.com/gradio-app/trackio/pull/438) [`0875ccd`](https://github.com/gradio-app/trackio/commit/0875ccd3d8a41b1376f64030f21cfe8cdcc73b05) - Add "share this view" functionality. Thanks @qgallouedec!
|
| 353 |
+
- [#409](https://github.com/gradio-app/trackio/pull/409) [`9282403`](https://github.com/gradio-app/trackio/commit/9282403d8896d48679b0f888208a7ba5bdd4271a) - Add Apple Silicon GPU and system monitoring support. Thanks @znation!
|
| 354 |
+
- [#434](https://github.com/gradio-app/trackio/pull/434) [`4193223`](https://github.com/gradio-app/trackio/commit/41932230a3a2e1c16405dba08ecba5a42f11d1a8) - fix: table slider crash. Thanks @Saba9!
|
| 355 |
+
|
| 356 |
+
### Fixes
|
| 357 |
+
|
| 358 |
+
- [#441](https://github.com/gradio-app/trackio/pull/441) [`3a2d11d`](https://github.com/gradio-app/trackio/commit/3a2d11dab0b4b37c925abc30ef84b0e2910321ee) - preserve x-axis step when toggling run checkboxes. Thanks @Saba9!
|
| 359 |
+
|
| 360 |
+
## 0.17.0
|
| 361 |
+
|
| 362 |
+
### Features
|
| 363 |
+
|
| 364 |
+
- [#428](https://github.com/gradio-app/trackio/pull/428) [`f7dd1ce`](https://github.com/gradio-app/trackio/commit/f7dd1ce2dc8a1936f9983467fcbcf93bfef01e09) - feat: add ability to rename runs. Thanks @Saba9!
|
| 365 |
+
- [#437](https://github.com/gradio-app/trackio/pull/437) [`2727c0b`](https://github.com/gradio-app/trackio/commit/2727c0b0755f48f7f186162ea45185c98f6b5516) - Add markdown reports across Trackio. Thanks @abidlabs!
|
| 366 |
+
- [#427](https://github.com/gradio-app/trackio/pull/427) [`5aeb9ed`](https://github.com/gradio-app/trackio/commit/5aeb9edcfd2068d309d9d64f172dcbcc327be1ab) - Make Trackio logging much more robust. Thanks @abidlabs!
|
| 367 |
+
|
| 368 |
+
## 0.16.1
|
| 369 |
+
|
| 370 |
+
### Features
|
| 371 |
+
|
| 372 |
+
- [#431](https://github.com/gradio-app/trackio/pull/431) [`c7ce55b`](https://github.com/gradio-app/trackio/commit/c7ce55b14dd5eb0c2165fb15df17dd60721c9325) - Lazy load the UI when trackio is imported. Thanks @abidlabs!
|
| 373 |
+
|
| 374 |
+
## 0.16.0
|
| 375 |
+
|
| 376 |
+
### Features
|
| 377 |
+
|
| 378 |
+
- [#426](https://github.com/gradio-app/trackio/pull/426) [`ead4dc8`](https://github.com/gradio-app/trackio/commit/ead4dc8e74ee2d8e47d61bca0a7668456acf49be) - Fix redundant double rendering of group checkboxes. Thanks @abidlabs!
|
| 379 |
+
- [#413](https://github.com/gradio-app/trackio/pull/413) [`39c4750`](https://github.com/gradio-app/trackio/commit/39c4750951d554ba6eb4d58847c6bb444b2891a8) - Check `dist-packages` when checking for source installation. Thanks @sergiopaniego!
|
| 380 |
+
- [#423](https://github.com/gradio-app/trackio/pull/423) [`2e52ab3`](https://github.com/gradio-app/trackio/commit/2e52ab303e3041718a6a56fbf84d0848aca9ad67) - Fix legend outline visibility issue. Thanks @Raghunath-Balaji!
|
| 381 |
+
- [#407](https://github.com/gradio-app/trackio/pull/407) [`c8a384d`](https://github.com/gradio-app/trackio/commit/c8a384ddfe5a295cecf862a26178d40e48acb424) - Fix pytests that were failling locally on MacOS. Thanks @abidlabs!
|
| 382 |
+
- [#405](https://github.com/gradio-app/trackio/pull/405) [`35aae4e`](https://github.com/gradio-app/trackio/commit/35aae4e3aa3e2b2888887528478b9dc6a9808bda) - Add conditional padding for HF Space dashboard when not in iframe. Thanks @znation!
|
| 383 |
+
|
| 384 |
+
## 0.15.0
|
| 385 |
+
|
| 386 |
+
### Features
|
| 387 |
+
|
| 388 |
+
- [#397](https://github.com/gradio-app/trackio/pull/397) [`6b38ad0`](https://github.com/gradio-app/trackio/commit/6b38ad02e5d73a0df49c4eede7e91331282ece04) - Adds `--host` cli option support. Thanks @abidlabs!
|
| 389 |
+
- [#396](https://github.com/gradio-app/trackio/pull/396) [`4a4d1ab`](https://github.com/gradio-app/trackio/commit/4a4d1ab85e63d923132a3fa7afa5d90e16431bec) - Fix run selection issue. Thanks @abidlabs!
|
| 390 |
+
- [#394](https://github.com/gradio-app/trackio/pull/394) [`c47a3a3`](https://github.com/gradio-app/trackio/commit/c47a3a31f8c4b83bce1aa7fc22eeba3d9021ad3d) - Add wandb-compatible API for trackio. Thanks @abidlabs!
|
| 391 |
+
- [#378](https://github.com/gradio-app/trackio/pull/378) [`b02046a`](https://github.com/gradio-app/trackio/commit/b02046a5b0dad7c9854e099a87f884afba4aecb2) - Add JSON export button for line plots and upgrade gradio dependency. Thanks @JamshedAli18!
|
| 392 |
+
|
| 393 |
+
## 0.14.2
|
| 394 |
+
|
| 395 |
+
### Features
|
| 396 |
+
|
| 397 |
+
- [#386](https://github.com/gradio-app/trackio/pull/386) [`f9452cd`](https://github.com/gradio-app/trackio/commit/f9452cdb8f0819368f3610f7ac0ed08957305275) - Fixing some issues related to deployed Trackio Spaces. Thanks @abidlabs!
|
| 398 |
+
|
| 399 |
+
## 0.14.1
|
| 400 |
+
|
| 401 |
+
### Features
|
| 402 |
+
|
| 403 |
+
- [#382](https://github.com/gradio-app/trackio/pull/382) [`44fe9bb`](https://github.com/gradio-app/trackio/commit/44fe9bb264fb2aafb0ec302ff15227c045819a2c) - Fix app file path when Trackio is not installed from source. Thanks @abidlabs!
|
| 404 |
+
- [#380](https://github.com/gradio-app/trackio/pull/380) [`c3f4cff`](https://github.com/gradio-app/trackio/commit/c3f4cff74bc5676e812773d8571454894fcdc7cc) - Add CLI commands for querying projects, runs, and metrics. Thanks @abidlabs!
|
| 405 |
+
|
| 406 |
+
## 0.14.0
|
| 407 |
+
|
| 408 |
+
### Features
|
| 409 |
+
|
| 410 |
+
- [#377](https://github.com/gradio-app/trackio/pull/377) [`5c5015b`](https://github.com/gradio-app/trackio/commit/5c5015b68c85c5de51111dad983f735c27b9a05f) - fixed wrapping issue in Runs table. Thanks @gaganchapa!
|
| 411 |
+
- [#374](https://github.com/gradio-app/trackio/pull/374) [`388e26b`](https://github.com/gradio-app/trackio/commit/388e26b9e9f24cd7ad203affe9b709be885b3d24) - Save Optimized Parquet files. Thanks @lhoestq!
|
| 412 |
+
- [#371](https://github.com/gradio-app/trackio/pull/371) [`fbace9c`](https://github.com/gradio-app/trackio/commit/fbace9cd7732c166f34d268f54b05bb06846cc5d) - Add GPU metrics logging. Thanks @kashif!
|
| 413 |
+
- [#367](https://github.com/gradio-app/trackio/pull/367) [`862840c`](https://github.com/gradio-app/trackio/commit/862840c13e30fc960cbee5b9eac4d3c25beba9de) - Add option to only show latest run, and fix the double logo issue. Thanks @abidlabs!
|
| 414 |
+
|
| 415 |
+
## 0.13.1
|
| 416 |
+
|
| 417 |
+
### Features
|
| 418 |
+
|
| 419 |
+
- [#369](https://github.com/gradio-app/trackio/pull/369) [`767e9fe`](https://github.com/gradio-app/trackio/commit/767e9fe095d7c6ed102016caf927c1517fb8618c) - tiny pr removing unnecessary code. Thanks @abidlabs!
|
| 420 |
+
|
| 421 |
+
## 0.13.0
|
| 422 |
+
|
| 423 |
+
### Features
|
| 424 |
+
|
| 425 |
+
- [#358](https://github.com/gradio-app/trackio/pull/358) [`073715d`](https://github.com/gradio-app/trackio/commit/073715d1caf8282f68890117f09c3ac301205312) - Improvements to `trackio.sync()`. Thanks @abidlabs!
|
| 426 |
+
|
| 427 |
+
## 0.12.0
|
| 428 |
+
|
| 429 |
+
### Features
|
| 430 |
+
|
| 431 |
+
- [#357](https://github.com/gradio-app/trackio/pull/357) [`02ba815`](https://github.com/gradio-app/trackio/commit/02ba815358060f1966052de051a5bdb09702920e) - Redesign media and tables to show up on separate page. Thanks @abidlabs!
|
| 432 |
+
- [#359](https://github.com/gradio-app/trackio/pull/359) [`08fe9c9`](https://github.com/gradio-app/trackio/commit/08fe9c9ddd7fe99ee811555fdfb62df9ab88e939) - docs: Improve docstrings. Thanks @qgallouedec!
|
| 433 |
+
|
| 434 |
+
## 0.11.0
|
| 435 |
+
|
| 436 |
+
### Features
|
| 437 |
+
|
| 438 |
+
- [#355](https://github.com/gradio-app/trackio/pull/355) [`ea51f49`](https://github.com/gradio-app/trackio/commit/ea51f4954922f21be76ef828700420fe9a912c4b) - Color code run checkboxes and match with plot lines. Thanks @abidlabs!
|
| 439 |
+
- [#353](https://github.com/gradio-app/trackio/pull/353) [`8abe691`](https://github.com/gradio-app/trackio/commit/8abe6919aeefe21fc7a23af814883efbb037c21f) - Remove show_api from demo.launch. Thanks @sergiopaniego!
|
| 440 |
+
- [#351](https://github.com/gradio-app/trackio/pull/351) [`8a8957e`](https://github.com/gradio-app/trackio/commit/8a8957e530dd7908d1fef7f2df030303f808101f) - Add `trackio.save()`. Thanks @abidlabs!
|
| 441 |
+
|
| 442 |
+
## 0.10.0
|
| 443 |
+
|
| 444 |
+
### Features
|
| 445 |
+
|
| 446 |
+
- [#305](https://github.com/gradio-app/trackio/pull/305) [`e64883a`](https://github.com/gradio-app/trackio/commit/e64883a51f7b8b93f7d48b8afe55acdb62238b71) - bump to gradio 6.0, make `trackio` compatible, and fix related issues. Thanks @abidlabs!
|
| 447 |
+
|
| 448 |
+
## 0.9.1
|
| 449 |
+
|
| 450 |
+
### Features
|
| 451 |
+
|
| 452 |
+
- [#344](https://github.com/gradio-app/trackio/pull/344) [`7e01024`](https://github.com/gradio-app/trackio/commit/7e010241d9a34794e0ce0dc19c1a6f0cf94ba856) - Avoid redundant calls to /whoami-v2. Thanks @Wauplin!
|
| 453 |
+
|
| 454 |
+
## 0.9.0
|
| 455 |
+
|
| 456 |
+
### Features
|
| 457 |
+
|
| 458 |
+
- [#343](https://github.com/gradio-app/trackio/pull/343) [`51bea30`](https://github.com/gradio-app/trackio/commit/51bea30f2877adff8e6497466d3a799400a0a049) - Sync offline projects to Hugging Face spaces. Thanks @candemircan!
|
| 459 |
+
- [#341](https://github.com/gradio-app/trackio/pull/341) [`4fd841f`](https://github.com/gradio-app/trackio/commit/4fd841fa190e15071b02f6fba7683ef4f393a654) - Adds a basic UI test to `trackio`. Thanks @abidlabs!
|
| 460 |
+
- [#339](https://github.com/gradio-app/trackio/pull/339) [`011d91b`](https://github.com/gradio-app/trackio/commit/011d91bb6ae266516fd250a349285670a8049d05) - Allow customzing the trackio color palette. Thanks @abidlabs!
|
| 461 |
+
|
| 462 |
+
## 0.8.1
|
| 463 |
+
|
| 464 |
+
### Features
|
| 465 |
+
|
| 466 |
+
- [#336](https://github.com/gradio-app/trackio/pull/336) [`5f9f51d`](https://github.com/gradio-app/trackio/commit/5f9f51dac8677f240d7c42c3e3b2660a22aee138) - Support a list of `Trackio.Image` in a `trackio.Table` cell. Thanks @abidlabs!
|
| 467 |
+
|
| 468 |
+
## 0.8.0
|
| 469 |
+
|
| 470 |
+
### Features
|
| 471 |
+
|
| 472 |
+
- [#331](https://github.com/gradio-app/trackio/pull/331) [`2c02d0f`](https://github.com/gradio-app/trackio/commit/2c02d0fd0a5824160528782402bb0dd4083396d5) - Truncate table string values that are greater than 250 characters (configuirable via env variable). Thanks @abidlabs!
|
| 473 |
+
- [#324](https://github.com/gradio-app/trackio/pull/324) [`50b2122`](https://github.com/gradio-app/trackio/commit/50b2122e7965ac82a72e6cb3b7d048bc10a2a6b1) - Add log y-axis functionality to UI. Thanks @abidlabs!
|
| 474 |
+
- [#326](https://github.com/gradio-app/trackio/pull/326) [`61dc1f4`](https://github.com/gradio-app/trackio/commit/61dc1f40af2f545f8e70395ddf0dbb8aee6b60d5) - Fix: improve table rendering for metrics in Trackio Dashboard. Thanks @vigneshwaran!
|
| 475 |
+
- [#328](https://github.com/gradio-app/trackio/pull/328) [`6857cbb`](https://github.com/gradio-app/trackio/commit/6857cbbe557a59a4642f210ec42566d108294e63) - Support trackio.Table with trackio.Image columns. Thanks @abidlabs!
|
| 476 |
+
- [#323](https://github.com/gradio-app/trackio/pull/323) [`6857cbb`](https://github.com/gradio-app/trackio/commit/6857cbbe557a59a4642f210ec42566d108294e63) - add Trackio client implementations in Go, Rust, and JS. Thanks @vaibhav-research!
|
| 477 |
+
|
| 478 |
+
## 0.7.0
|
| 479 |
+
|
| 480 |
+
### Features
|
| 481 |
+
|
| 482 |
+
- [#277](https://github.com/gradio-app/trackio/pull/277) [`db35601`](https://github.com/gradio-app/trackio/commit/db35601b9c023423c4654c9909b8ab73e58737de) - fix: make grouped runs view reflect live updates. Thanks @Saba9!
|
| 483 |
+
- [#320](https://github.com/gradio-app/trackio/pull/320) [`24ae739`](https://github.com/gradio-app/trackio/commit/24ae73969b09fb3126acd2f91647cdfbf8cf72a1) - Add additional query parms for xmin, xmax, and smoothing. Thanks @abidlabs!
|
| 484 |
+
- [#270](https://github.com/gradio-app/trackio/pull/270) [`cd1dfc3`](https://github.com/gradio-app/trackio/commit/cd1dfc3dc641b4499ac6d4a1b066fa8e2b52c57b) - feature: add support for logging audio. Thanks @Saba9!
|
| 485 |
+
|
| 486 |
+
## 0.6.0
|
| 487 |
+
|
| 488 |
+
### Features
|
| 489 |
+
|
| 490 |
+
- [#309](https://github.com/gradio-app/trackio/pull/309) [`1df2353`](https://github.com/gradio-app/trackio/commit/1df23534d6c01938c8db9c0f584ffa23e8d6021d) - Add histogram support with wandb-compatible API. Thanks @abidlabs!
|
| 491 |
+
- [#315](https://github.com/gradio-app/trackio/pull/315) [`76ba060`](https://github.com/gradio-app/trackio/commit/76ba06055dc43ca8f03b79f3e72d761949bd19a8) - Add guards to avoid silent fails. Thanks @Xmaster6y!
|
| 492 |
+
- [#313](https://github.com/gradio-app/trackio/pull/313) [`a606b3e`](https://github.com/gradio-app/trackio/commit/a606b3e1c5edf3d4cf9f31bd50605226a5a1c5d0) - No longer prevent certain keys from being used. Instead, dunderify them to prevent collisions with internal usage. Thanks @abidlabs!
|
| 493 |
+
- [#317](https://github.com/gradio-app/trackio/pull/317) [`27370a5`](https://github.com/gradio-app/trackio/commit/27370a595d0dbdf7eebbe7159d2ba778f039da44) - quick fixes for trackio.histogram. Thanks @abidlabs!
|
| 494 |
+
- [#312](https://github.com/gradio-app/trackio/pull/312) [`aa0f3bf`](https://github.com/gradio-app/trackio/commit/aa0f3bf372e7a0dd592a38af699c998363830eeb) - Fix video logging by adding TRACKIO_DIR to allowed_paths. Thanks @abidlabs!
|
| 495 |
+
|
| 496 |
+
## 0.5.3
|
| 497 |
+
|
| 498 |
+
### Features
|
| 499 |
+
|
| 500 |
+
- [#300](https://github.com/gradio-app/trackio/pull/300) [`5e4cacf`](https://github.com/gradio-app/trackio/commit/5e4cacf2e7ce527b4ce60de3a5bc05d2c02c77fb) - Adds more environment variables to allow customization of Trackio dashboard. Thanks @abidlabs!
|
| 501 |
+
|
| 502 |
+
## 0.5.2
|
| 503 |
+
|
| 504 |
+
### Features
|
| 505 |
+
|
| 506 |
+
- [#293](https://github.com/gradio-app/trackio/pull/293) [`64afc28`](https://github.com/gradio-app/trackio/commit/64afc28d3ea1dfd821472dc6bf0b8ed35a9b74be) - Ensures that the TRACKIO_DIR environment variable is respected. Thanks @abidlabs!
|
| 507 |
+
- [#287](https://github.com/gradio-app/trackio/pull/287) [`cd3e929`](https://github.com/gradio-app/trackio/commit/cd3e9294320949e6b8b829239069a43d5d7ff4c1) - fix(sqlite): unify .sqlite extension, allow export when DBs exist, clean WAL sidecars on import. Thanks @vaibhav-research!
|
| 508 |
+
|
| 509 |
+
### Fixes
|
| 510 |
+
|
| 511 |
+
- [#291](https://github.com/gradio-app/trackio/pull/291) [`3b5adc3`](https://github.com/gradio-app/trackio/commit/3b5adc3d1f452dbab7a714d235f4974782f93730) - Fix the wheel build. Thanks @pngwn!
|
| 512 |
+
|
| 513 |
+
## 0.5.1
|
| 514 |
+
|
| 515 |
+
### Fixes
|
| 516 |
+
|
| 517 |
+
- [#278](https://github.com/gradio-app/trackio/pull/278) [`314c054`](https://github.com/gradio-app/trackio/commit/314c05438007ddfea3383e06fd19143e27468e2d) - Fix row orientation of metrics plots. Thanks @abidlabs!
|
trackio/__init__.py
ADDED
|
@@ -0,0 +1,1280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import atexit
|
| 2 |
+
import glob
|
| 3 |
+
import json
|
| 4 |
+
import logging
|
| 5 |
+
import os
|
| 6 |
+
import shutil
|
| 7 |
+
import warnings
|
| 8 |
+
import webbrowser
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
import huggingface_hub
|
| 13 |
+
from gradio_client import handle_file
|
| 14 |
+
from huggingface_hub import SpaceStorage
|
| 15 |
+
from huggingface_hub.errors import LocalTokenNotFoundError
|
| 16 |
+
|
| 17 |
+
from trackio import context_vars, deploy, utils
|
| 18 |
+
from trackio._unsupported import (
|
| 19 |
+
ArtifactTTL,
|
| 20 |
+
Classes,
|
| 21 |
+
Config,
|
| 22 |
+
EvalTable,
|
| 23 |
+
Graph,
|
| 24 |
+
JoinedTable,
|
| 25 |
+
Molecule,
|
| 26 |
+
Plotly,
|
| 27 |
+
Settings,
|
| 28 |
+
)
|
| 29 |
+
from trackio.alerts import AlertLevel
|
| 30 |
+
from trackio.api import Api
|
| 31 |
+
from trackio.apple_gpu import apple_gpu_available
|
| 32 |
+
from trackio.apple_gpu import log_apple_gpu as _log_apple_gpu
|
| 33 |
+
from trackio.artifact import Artifact
|
| 34 |
+
from trackio.cpu import cpu_available
|
| 35 |
+
from trackio.cpu import log_cpu as _log_cpu
|
| 36 |
+
from trackio.deploy import freeze, sync
|
| 37 |
+
from trackio.frontend_config import resolve_frontend_dir
|
| 38 |
+
from trackio.gpu import gpu_available
|
| 39 |
+
from trackio.gpu import log_gpu as _log_nvidia_gpu
|
| 40 |
+
from trackio.histogram import Histogram
|
| 41 |
+
from trackio.imports import import_csv, import_tf_events
|
| 42 |
+
from trackio.launch import launch_trackio_dashboard
|
| 43 |
+
from trackio.markdown import Markdown
|
| 44 |
+
from trackio.media import (
|
| 45 |
+
TrackioAudio,
|
| 46 |
+
TrackioHtml,
|
| 47 |
+
TrackioImage,
|
| 48 |
+
TrackioObject3D,
|
| 49 |
+
TrackioVideo,
|
| 50 |
+
get_project_media_path,
|
| 51 |
+
)
|
| 52 |
+
from trackio.references import (
|
| 53 |
+
ReferenceHandler,
|
| 54 |
+
ResolvedReference,
|
| 55 |
+
register_reference_handler,
|
| 56 |
+
)
|
| 57 |
+
from trackio.registry import Collection as Collection
|
| 58 |
+
from trackio.registry import Registry as Registry
|
| 59 |
+
from trackio.remote_client import RemoteClient
|
| 60 |
+
from trackio.run import Run
|
| 61 |
+
from trackio.server import TrackioDashboardApp, build_starlette_app_only
|
| 62 |
+
from trackio.sqlite_storage import SQLiteStorage
|
| 63 |
+
from trackio.table import Table
|
| 64 |
+
from trackio.trace import Trace
|
| 65 |
+
from trackio.typehints import UploadEntry
|
| 66 |
+
from trackio.utils import TRACKIO_DIR, TRACKIO_LOGO_DIR, _emit_nonfatal_warning
|
| 67 |
+
|
| 68 |
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
| 69 |
+
|
| 70 |
+
__version__ = json.loads(Path(__file__).parent.joinpath("package.json").read_text())[
|
| 71 |
+
"version"
|
| 72 |
+
]
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class _TupleNoPrint(tuple):
|
| 76 |
+
def __repr__(self) -> str:
|
| 77 |
+
return ""
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
__all__ = [
|
| 81 |
+
"init",
|
| 82 |
+
"log",
|
| 83 |
+
"log_system",
|
| 84 |
+
"log_gpu",
|
| 85 |
+
"log_artifact",
|
| 86 |
+
"use_artifact",
|
| 87 |
+
"log_cpu",
|
| 88 |
+
"finish",
|
| 89 |
+
"alert",
|
| 90 |
+
"AlertLevel",
|
| 91 |
+
"show",
|
| 92 |
+
"sync",
|
| 93 |
+
"freeze",
|
| 94 |
+
"delete_project",
|
| 95 |
+
"import_csv",
|
| 96 |
+
"import_tf_events",
|
| 97 |
+
"save",
|
| 98 |
+
"Artifact",
|
| 99 |
+
"ArtifactTTL",
|
| 100 |
+
"Classes",
|
| 101 |
+
"Config",
|
| 102 |
+
"EvalTable",
|
| 103 |
+
"Graph",
|
| 104 |
+
"JoinedTable",
|
| 105 |
+
"Molecule",
|
| 106 |
+
"Plotly",
|
| 107 |
+
"Settings",
|
| 108 |
+
"ReferenceHandler",
|
| 109 |
+
"ResolvedReference",
|
| 110 |
+
"register_reference_handler",
|
| 111 |
+
"Image",
|
| 112 |
+
"Video",
|
| 113 |
+
"Audio",
|
| 114 |
+
"Html",
|
| 115 |
+
"Object3D",
|
| 116 |
+
"Table",
|
| 117 |
+
"Trace",
|
| 118 |
+
"Histogram",
|
| 119 |
+
"Markdown",
|
| 120 |
+
"Api",
|
| 121 |
+
"TRACKIO_LOGO_DIR",
|
| 122 |
+
]
|
| 123 |
+
|
| 124 |
+
Audio = TrackioAudio
|
| 125 |
+
Image = TrackioImage
|
| 126 |
+
Video = TrackioVideo
|
| 127 |
+
Html = TrackioHtml
|
| 128 |
+
Object3D = TrackioObject3D
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
config = {}
|
| 132 |
+
|
| 133 |
+
_atexit_registered = False
|
| 134 |
+
_spaces_created_this_session: set[str] = set()
|
| 135 |
+
_projects_notified_auto_log_hw: set[str] = set()
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def _cleanup_current_run():
|
| 139 |
+
run = context_vars.current_run.get()
|
| 140 |
+
if run is not None:
|
| 141 |
+
try:
|
| 142 |
+
run.finish()
|
| 143 |
+
except Exception:
|
| 144 |
+
pass
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _safe_get_runs_for_init(
|
| 148 |
+
project: str,
|
| 149 |
+
space_id: str | None,
|
| 150 |
+
server_base_url: str | None,
|
| 151 |
+
write_token: str | None,
|
| 152 |
+
resume: str,
|
| 153 |
+
remote_client: RemoteClient | None = None,
|
| 154 |
+
check_existing_for_never: bool = False,
|
| 155 |
+
) -> list[str]:
|
| 156 |
+
if space_id is not None or server_base_url is not None:
|
| 157 |
+
if resume == "never" and not check_existing_for_never:
|
| 158 |
+
return []
|
| 159 |
+
if remote_client is not None:
|
| 160 |
+
source = space_id or server_base_url
|
| 161 |
+
try:
|
| 162 |
+
runs = remote_client.predict(
|
| 163 |
+
project=project, api_name="/get_runs_for_project"
|
| 164 |
+
)
|
| 165 |
+
return runs if isinstance(runs, list) else []
|
| 166 |
+
except Exception as e:
|
| 167 |
+
_emit_nonfatal_warning(
|
| 168 |
+
f"trackio.init() could not inspect existing runs for project '{project}' on '{source}': {e}. Continuing without resume metadata."
|
| 169 |
+
)
|
| 170 |
+
return []
|
| 171 |
+
try:
|
| 172 |
+
return SQLiteStorage.get_runs(project)
|
| 173 |
+
except Exception as e:
|
| 174 |
+
_emit_nonfatal_warning(
|
| 175 |
+
f"trackio.init() could not inspect existing runs for project '{project}': {e}. Continuing without resume metadata."
|
| 176 |
+
)
|
| 177 |
+
return []
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def _migrate_local_project_for_init(project: str) -> None:
|
| 181 |
+
if utils.get_storage_mode() == "jsonl":
|
| 182 |
+
return
|
| 183 |
+
if not SQLiteStorage.get_project_db_path(project).exists():
|
| 184 |
+
return
|
| 185 |
+
try:
|
| 186 |
+
SQLiteStorage.init_db(project)
|
| 187 |
+
except Exception as e:
|
| 188 |
+
_emit_nonfatal_warning(
|
| 189 |
+
f"trackio.init() could not migrate the local database for project '{project}': {e}."
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _safe_get_latest_run_for_init(
|
| 194 |
+
project: str,
|
| 195 |
+
name: str,
|
| 196 |
+
space_id: str | None = None,
|
| 197 |
+
server_base_url: str | None = None,
|
| 198 |
+
write_token: str | None = None,
|
| 199 |
+
remote_client: RemoteClient | None = None,
|
| 200 |
+
) -> dict | None:
|
| 201 |
+
if (
|
| 202 |
+
space_id is not None or server_base_url is not None
|
| 203 |
+
) and remote_client is not None:
|
| 204 |
+
source = space_id or server_base_url
|
| 205 |
+
try:
|
| 206 |
+
runs = remote_client.predict(
|
| 207 |
+
project=project, api_name="/get_runs_for_project"
|
| 208 |
+
)
|
| 209 |
+
if not isinstance(runs, list):
|
| 210 |
+
return None
|
| 211 |
+
matches = [r for r in runs if isinstance(r, dict) and r.get("name") == name]
|
| 212 |
+
if not matches:
|
| 213 |
+
return None
|
| 214 |
+
matches.sort(key=lambda r: r.get("created_at") or "", reverse=True)
|
| 215 |
+
return matches[0]
|
| 216 |
+
except Exception as e:
|
| 217 |
+
_emit_nonfatal_warning(
|
| 218 |
+
f"trackio.init() could not inspect existing runs for project '{project}' on '{source}': {e}. Continuing without resume metadata."
|
| 219 |
+
)
|
| 220 |
+
return None
|
| 221 |
+
try:
|
| 222 |
+
return SQLiteStorage.get_latest_run_record_by_name(project, name)
|
| 223 |
+
except Exception as e:
|
| 224 |
+
_emit_nonfatal_warning(
|
| 225 |
+
f"trackio.init() could not inspect existing runs for project '{project}': {e}. Continuing without resume metadata."
|
| 226 |
+
)
|
| 227 |
+
return None
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _safe_get_last_step_for_init(
|
| 231 |
+
project: str,
|
| 232 |
+
run_name: str,
|
| 233 |
+
space_id: str | None,
|
| 234 |
+
server_base_url: str | None,
|
| 235 |
+
write_token: str | None,
|
| 236 |
+
resumed: bool,
|
| 237 |
+
run_id: str | None = None,
|
| 238 |
+
remote_client: RemoteClient | None = None,
|
| 239 |
+
) -> int | None:
|
| 240 |
+
if not resumed:
|
| 241 |
+
return None
|
| 242 |
+
if (
|
| 243 |
+
space_id is not None or server_base_url is not None
|
| 244 |
+
) and remote_client is not None:
|
| 245 |
+
source = space_id or server_base_url
|
| 246 |
+
try:
|
| 247 |
+
summary_kwargs: dict[str, Any] = {
|
| 248 |
+
"project": project,
|
| 249 |
+
"api_name": "/get_run_summary",
|
| 250 |
+
}
|
| 251 |
+
if run_id is not None:
|
| 252 |
+
summary_kwargs["run_id"] = run_id
|
| 253 |
+
else:
|
| 254 |
+
summary_kwargs["run"] = run_name
|
| 255 |
+
summary = remote_client.predict(**summary_kwargs)
|
| 256 |
+
if isinstance(summary, dict):
|
| 257 |
+
last_step = summary.get("last_step")
|
| 258 |
+
return last_step if isinstance(last_step, int) else None
|
| 259 |
+
return None
|
| 260 |
+
except Exception as e:
|
| 261 |
+
_emit_nonfatal_warning(
|
| 262 |
+
f"trackio.init() could not recover the previous step for run '{run_name}' on '{source}': {e}. Continuing from step 0."
|
| 263 |
+
)
|
| 264 |
+
return None
|
| 265 |
+
try:
|
| 266 |
+
return SQLiteStorage.get_max_step_for_run(project, run_name, run_id=run_id)
|
| 267 |
+
except Exception as e:
|
| 268 |
+
_emit_nonfatal_warning(
|
| 269 |
+
f"trackio.init() could not recover the previous step for run '{run_name}': {e}. Continuing from step 0."
|
| 270 |
+
)
|
| 271 |
+
return None
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def init(
|
| 275 |
+
project: str,
|
| 276 |
+
name: str | None = None,
|
| 277 |
+
group: str | None = None,
|
| 278 |
+
space_id: str | None = None,
|
| 279 |
+
server_url: str | None = None,
|
| 280 |
+
space_storage: SpaceStorage | None = None,
|
| 281 |
+
dataset_id: str | None = None,
|
| 282 |
+
bucket_id: str | None = None,
|
| 283 |
+
config: dict | None = None,
|
| 284 |
+
resume: str = "never",
|
| 285 |
+
settings: Any = None,
|
| 286 |
+
private: bool | None = None,
|
| 287 |
+
embed: bool = True,
|
| 288 |
+
auto_log_gpu: bool | None = None,
|
| 289 |
+
gpu_log_interval: float = 10.0,
|
| 290 |
+
auto_log_cpu: bool | None = None,
|
| 291 |
+
cpu_log_interval: float = 10.0,
|
| 292 |
+
webhook_url: str | None = None,
|
| 293 |
+
webhook_min_level: AlertLevel | str | None = None,
|
| 294 |
+
) -> Run:
|
| 295 |
+
"""
|
| 296 |
+
Creates a new Trackio project and returns a [`Run`] object.
|
| 297 |
+
|
| 298 |
+
Args:
|
| 299 |
+
project (`str`):
|
| 300 |
+
The name of the project (can be an existing project to continue tracking or
|
| 301 |
+
a new project to start tracking from scratch).
|
| 302 |
+
name (`str`, *optional*):
|
| 303 |
+
The name of the run (if not provided, a default name will be generated).
|
| 304 |
+
group (`str`, *optional*):
|
| 305 |
+
The name of the group which this run belongs to in order to help organize
|
| 306 |
+
related runs together. You can toggle the entire group's visibility in the
|
| 307 |
+
dashboard.
|
| 308 |
+
space_id (`str`, *optional*):
|
| 309 |
+
If provided, the project will be logged to a Hugging Face Space instead of
|
| 310 |
+
a local directory. Should be a complete Space name like
|
| 311 |
+
`"username/reponame"` or `"orgname/reponame"`, or just `"reponame"` in which
|
| 312 |
+
case the Space will be created in the currently-logged-in Hugging Face
|
| 313 |
+
user's namespace. If the Space does not exist, it will be created. If the
|
| 314 |
+
Space already exists, the project will be logged to it. Can also be set
|
| 315 |
+
via the `TRACKIO_SPACE_ID` environment variable. You cannot log to a
|
| 316 |
+
Space that has been **frozen** (converted to the static SDK); use
|
| 317 |
+
``trackio.sync(..., sdk="static")`` only after you are done logging.
|
| 318 |
+
Takes precedence over `server_url` and `TRACKIO_SERVER_URL` when more than
|
| 319 |
+
one is set.
|
| 320 |
+
server_url (`str`, *optional*):
|
| 321 |
+
Base URL of a self-hosted Trackio server (``http://`` or ``https://``), or the
|
| 322 |
+
write-access URL from ``trackio.show()`` which may include a ``write_token`` query
|
| 323 |
+
parameter. The client sends that token on each request (``X-Trackio-Write-Token``);
|
| 324 |
+
you can also set ``TRACKIO_WRITE_TOKEN`` instead of embedding the token in the URL.
|
| 325 |
+
When set, metrics are sent to that server over HTTP instead of creating or syncing
|
| 326 |
+
to a Hugging Face Space. Can also be set via the ``TRACKIO_SERVER_URL`` environment
|
| 327 |
+
variable. Ignored when ``space_id`` or ``TRACKIO_SPACE_ID`` is set.
|
| 328 |
+
space_storage ([`~huggingface_hub.SpaceStorage`], *optional*):
|
| 329 |
+
Choice of persistent storage tier.
|
| 330 |
+
dataset_id (`str`, *optional*):
|
| 331 |
+
Deprecated: persisting trackio data to a Hugging Face Dataset will be
|
| 332 |
+
removed in a future version of trackio. Use `bucket_id` (a Hugging
|
| 333 |
+
Face Bucket) instead.
|
| 334 |
+
bucket_id (`str`, *optional*):
|
| 335 |
+
The ID of the Hugging Face Bucket to use for metric persistence. By default,
|
| 336 |
+
when a `space_id` is provided and `bucket_id` is not explicitly set, a
|
| 337 |
+
bucket is auto-generated from the space_id. Buckets provide
|
| 338 |
+
S3-like storage without git overhead - the SQLite database is stored directly
|
| 339 |
+
via `hf-mount` in the Space. Specify a Bucket with name like
|
| 340 |
+
`"username/bucketname"` or just `"bucketname"`.
|
| 341 |
+
config (`dict`, *optional*):
|
| 342 |
+
A dictionary of configuration options. Provided for compatibility with
|
| 343 |
+
`wandb.init()`.
|
| 344 |
+
resume (`str`, *optional*, defaults to `"never"`):
|
| 345 |
+
Controls how to handle resuming a run. Can be one of:
|
| 346 |
+
|
| 347 |
+
- `"must"`: Must resume the run with the given name, raises error if run
|
| 348 |
+
doesn't exist
|
| 349 |
+
- `"allow"`: Resume the run if it exists, otherwise create a new run
|
| 350 |
+
- `"never"`: Never resume a run, always create a new one
|
| 351 |
+
private (`bool`, *optional*):
|
| 352 |
+
Whether to make the Space private. If None (default), the repo will be
|
| 353 |
+
public unless the organization's default is private. This value is ignored
|
| 354 |
+
if the repo already exists.
|
| 355 |
+
settings (`Any`, *optional*):
|
| 356 |
+
Not used. Provided for compatibility with `wandb.init()`.
|
| 357 |
+
embed (`bool`, *optional*, defaults to `True`):
|
| 358 |
+
If running inside a Jupyter/Colab notebook, whether the dashboard should
|
| 359 |
+
automatically be embedded in the cell when trackio.init() is called. For
|
| 360 |
+
local runs, this launches a local Trackio dashboard and embeds it. For Space runs,
|
| 361 |
+
this embeds the Space URL. In Colab, the local dashboard will be accessible
|
| 362 |
+
via a public share URL when `share=True`.
|
| 363 |
+
auto_log_gpu (`bool` or `None`, *optional*, defaults to `None`):
|
| 364 |
+
Controls automatic GPU metrics logging. If `None` (default), GPU logging
|
| 365 |
+
is automatically enabled when `nvidia-ml-py` is installed and an NVIDIA
|
| 366 |
+
GPU or Apple M series is detected. Set to `True` to force enable or
|
| 367 |
+
`False` to disable.
|
| 368 |
+
gpu_log_interval (`float`, *optional*, defaults to `10.0`):
|
| 369 |
+
The interval in seconds between automatic GPU metric logs.
|
| 370 |
+
Only used when `auto_log_gpu=True`.
|
| 371 |
+
auto_log_cpu (`bool` or `None`, *optional*, defaults to `None`):
|
| 372 |
+
Controls automatic CPU and RAM metrics logging (utilization, memory,
|
| 373 |
+
disk I/O, network I/O, and sensors). If `None` (default), CPU logging
|
| 374 |
+
is automatically enabled when `psutil` is installed. Set to `True` to
|
| 375 |
+
force enable or `False` to disable.
|
| 376 |
+
cpu_log_interval (`float`, *optional*, defaults to `10.0`):
|
| 377 |
+
The interval in seconds between automatic CPU metric logs.
|
| 378 |
+
Only used when CPU auto-logging is enabled.
|
| 379 |
+
webhook_url (`str`, *optional*):
|
| 380 |
+
A webhook URL to POST alert payloads to when `trackio.alert()` is
|
| 381 |
+
called. Supports Slack and Discord webhook URLs natively (payloads
|
| 382 |
+
are formatted automatically). Can also be set via the
|
| 383 |
+
`TRACKIO_WEBHOOK_URL` environment variable. Individual alerts can
|
| 384 |
+
override this URL by passing `webhook_url` to `trackio.alert()`.
|
| 385 |
+
webhook_min_level (`AlertLevel` or `str`, *optional*):
|
| 386 |
+
Minimum alert level that should trigger webhook delivery.
|
| 387 |
+
For example, `AlertLevel.WARN` sends only `WARN` and `ERROR`
|
| 388 |
+
alerts to the webhook destination. Can also be set via
|
| 389 |
+
`TRACKIO_WEBHOOK_MIN_LEVEL`.
|
| 390 |
+
Returns:
|
| 391 |
+
`Run`: A [`Run`] object that can be used to log metrics and finish the run.
|
| 392 |
+
"""
|
| 393 |
+
SQLiteStorage.validate_project_name(project)
|
| 394 |
+
|
| 395 |
+
if settings is not None:
|
| 396 |
+
_emit_nonfatal_warning(
|
| 397 |
+
"* Warning: settings is not used. Provided for compatibility with wandb.init(). Please create an issue at: https://github.com/gradio-app/trackio/issues if you need a specific feature implemented."
|
| 398 |
+
)
|
| 399 |
+
|
| 400 |
+
previous_run = context_vars.current_run.get()
|
| 401 |
+
if previous_run is not None:
|
| 402 |
+
try:
|
| 403 |
+
previous_run.finish()
|
| 404 |
+
except Exception as e:
|
| 405 |
+
_emit_nonfatal_warning(
|
| 406 |
+
f"trackio.init() could not finish the previous run '{previous_run.name}': {e}. Continuing with new run."
|
| 407 |
+
)
|
| 408 |
+
context_vars.current_run.set(None)
|
| 409 |
+
|
| 410 |
+
bucket_id_was_explicit = bucket_id is not None
|
| 411 |
+
space_id, server_url = utils.resolve_space_id_and_server_url(space_id, server_url)
|
| 412 |
+
if bucket_id is None and utils.on_spaces():
|
| 413 |
+
bucket_id = os.environ.get("TRACKIO_BUCKET_ID")
|
| 414 |
+
if server_url is not None and not server_url.startswith(("http://", "https://")):
|
| 415 |
+
raise ValueError(
|
| 416 |
+
f"`server_url` must be a full URL starting with http:// or https://, got: {server_url!r}"
|
| 417 |
+
)
|
| 418 |
+
server_base_url: str | None = None
|
| 419 |
+
write_token_resolved: str | None = None
|
| 420 |
+
if server_url is not None:
|
| 421 |
+
server_base_url, tok = utils.parse_trackio_server_url(server_url)
|
| 422 |
+
write_token_resolved = tok or os.environ.get("TRACKIO_WRITE_TOKEN")
|
| 423 |
+
if not write_token_resolved:
|
| 424 |
+
raise ValueError(
|
| 425 |
+
"Self-hosted logging requires a write token: add write_token to the server URL, "
|
| 426 |
+
"or set the TRACKIO_WRITE_TOKEN environment variable."
|
| 427 |
+
)
|
| 428 |
+
if server_url is not None and (dataset_id is not None or bucket_id is not None):
|
| 429 |
+
raise ValueError(
|
| 430 |
+
"`dataset_id` and `bucket_id` are Hugging Face Spaces concepts and are not "
|
| 431 |
+
"compatible with `server_url`. Configure storage on the self-hosted server."
|
| 432 |
+
)
|
| 433 |
+
if space_id is None and dataset_id is not None:
|
| 434 |
+
raise ValueError("Must provide a `space_id` when `dataset_id` is provided.")
|
| 435 |
+
if dataset_id is not None and bucket_id is not None:
|
| 436 |
+
raise ValueError("Cannot provide both `dataset_id` and `bucket_id`.")
|
| 437 |
+
try:
|
| 438 |
+
space_id, dataset_id, bucket_id = utils.preprocess_space_and_dataset_ids(
|
| 439 |
+
space_id, dataset_id, bucket_id
|
| 440 |
+
)
|
| 441 |
+
if (
|
| 442 |
+
space_id is not None
|
| 443 |
+
and dataset_id is None
|
| 444 |
+
and bucket_id is not None
|
| 445 |
+
and not bucket_id_was_explicit
|
| 446 |
+
and not utils.on_spaces()
|
| 447 |
+
):
|
| 448 |
+
bucket_id = deploy.resolve_auto_bucket_id(space_id, bucket_id)
|
| 449 |
+
except LocalTokenNotFoundError as e:
|
| 450 |
+
raise LocalTokenNotFoundError(
|
| 451 |
+
f"You must be logged in to Hugging Face locally when `space_id` is provided to deploy to a Space. {e}"
|
| 452 |
+
) from e
|
| 453 |
+
|
| 454 |
+
if space_id is None and bucket_id is not None:
|
| 455 |
+
_emit_nonfatal_warning(
|
| 456 |
+
"trackio.init() has `bucket_id` set but `space_id` is None: metrics will be logged "
|
| 457 |
+
"locally only. Pass `space_id` to create or use a Hugging Face Space, which will be "
|
| 458 |
+
"attached to the Hugging Face Bucket.",
|
| 459 |
+
UserWarning,
|
| 460 |
+
stacklevel=2,
|
| 461 |
+
)
|
| 462 |
+
|
| 463 |
+
if space_id is not None:
|
| 464 |
+
deploy.raise_if_space_is_frozen_for_logging(space_id)
|
| 465 |
+
|
| 466 |
+
remote_source = space_id or server_base_url
|
| 467 |
+
|
| 468 |
+
if remote_source is not None:
|
| 469 |
+
url = remote_source
|
| 470 |
+
context_vars.current_server.set(url)
|
| 471 |
+
if space_id is not None:
|
| 472 |
+
context_vars.current_space_id.set(space_id)
|
| 473 |
+
context_vars.current_server_write_token.set(None)
|
| 474 |
+
else:
|
| 475 |
+
context_vars.current_space_id.set(None)
|
| 476 |
+
context_vars.current_server_write_token.set(write_token_resolved)
|
| 477 |
+
else:
|
| 478 |
+
url = None
|
| 479 |
+
context_vars.current_server.set(None)
|
| 480 |
+
context_vars.current_space_id.set(None)
|
| 481 |
+
context_vars.current_server_write_token.set(None)
|
| 482 |
+
|
| 483 |
+
_should_embed_local = False
|
| 484 |
+
|
| 485 |
+
newly_created_space = False
|
| 486 |
+
if space_id is not None and space_id in _spaces_created_this_session:
|
| 487 |
+
if deploy.space_is_running(space_id):
|
| 488 |
+
_spaces_created_this_session.discard(space_id)
|
| 489 |
+
else:
|
| 490 |
+
newly_created_space = True
|
| 491 |
+
|
| 492 |
+
if (
|
| 493 |
+
context_vars.current_project.get() is None
|
| 494 |
+
or context_vars.current_project.get() != project
|
| 495 |
+
):
|
| 496 |
+
print(f"* Trackio project initialized: {project}")
|
| 497 |
+
|
| 498 |
+
if bucket_id is not None:
|
| 499 |
+
if utils.on_spaces():
|
| 500 |
+
os.environ["TRACKIO_BUCKET_ID"] = bucket_id
|
| 501 |
+
bucket_url = f"https://huggingface.co/buckets/{bucket_id}"
|
| 502 |
+
print(
|
| 503 |
+
f"* Trackio metrics will be synced to Hugging Face Bucket: {bucket_url}"
|
| 504 |
+
)
|
| 505 |
+
elif dataset_id is not None:
|
| 506 |
+
if utils.on_spaces():
|
| 507 |
+
os.environ["TRACKIO_DATASET_ID"] = dataset_id
|
| 508 |
+
print(
|
| 509 |
+
f"* Trackio metrics will be synced to Hugging Face Dataset: {dataset_id}"
|
| 510 |
+
)
|
| 511 |
+
if remote_source is None:
|
| 512 |
+
print(f"* Trackio metrics logged to: {TRACKIO_DIR}")
|
| 513 |
+
_should_embed_local = embed and utils.is_in_notebook()
|
| 514 |
+
if not _should_embed_local:
|
| 515 |
+
utils.print_dashboard_instructions(project)
|
| 516 |
+
elif server_base_url is not None:
|
| 517 |
+
print(
|
| 518 |
+
f"* Trackio metrics will be sent to self-hosted server: {server_base_url}"
|
| 519 |
+
)
|
| 520 |
+
if utils.is_in_notebook() and embed:
|
| 521 |
+
utils.embed_url_in_notebook(server_base_url)
|
| 522 |
+
else:
|
| 523 |
+
try:
|
| 524 |
+
if deploy.create_space_if_not_exists(
|
| 525 |
+
space_id,
|
| 526 |
+
space_storage,
|
| 527 |
+
dataset_id,
|
| 528 |
+
bucket_id,
|
| 529 |
+
private,
|
| 530 |
+
):
|
| 531 |
+
_spaces_created_this_session.add(space_id)
|
| 532 |
+
newly_created_space = True
|
| 533 |
+
user_name, space_name = space_id.split("/")
|
| 534 |
+
space_url = deploy.SPACE_HOST_URL.format(
|
| 535 |
+
user_name=user_name, space_name=space_name
|
| 536 |
+
)
|
| 537 |
+
if utils.is_in_notebook() and embed:
|
| 538 |
+
utils.embed_url_in_notebook(space_url)
|
| 539 |
+
except Exception as e:
|
| 540 |
+
_emit_nonfatal_warning(
|
| 541 |
+
f"trackio.init() could not prepare Space '{space_id}': {e}. Logging will continue in local fallback mode until the Space is reachable."
|
| 542 |
+
)
|
| 543 |
+
context_vars.current_project.set(project)
|
| 544 |
+
|
| 545 |
+
remote_client = None
|
| 546 |
+
if space_id is not None and not newly_created_space:
|
| 547 |
+
try:
|
| 548 |
+
remote_client = RemoteClient(
|
| 549 |
+
space_id,
|
| 550 |
+
hf_token=huggingface_hub.utils.get_token(),
|
| 551 |
+
verbose=False,
|
| 552 |
+
)
|
| 553 |
+
except Exception as e:
|
| 554 |
+
_emit_nonfatal_warning(
|
| 555 |
+
f"trackio.init() could not create a remote client for Space '{space_id}': {e}. Continuing with local fallback metadata lookups."
|
| 556 |
+
)
|
| 557 |
+
elif server_base_url is not None:
|
| 558 |
+
try:
|
| 559 |
+
remote_client = RemoteClient(
|
| 560 |
+
server_base_url,
|
| 561 |
+
hf_token=None,
|
| 562 |
+
write_token=write_token_resolved,
|
| 563 |
+
verbose=False,
|
| 564 |
+
)
|
| 565 |
+
except Exception as e:
|
| 566 |
+
_emit_nonfatal_warning(
|
| 567 |
+
f"trackio.init() could not create a remote client for '{server_base_url}': {e}. Continuing with local fallback metadata lookups."
|
| 568 |
+
)
|
| 569 |
+
|
| 570 |
+
_migrate_local_project_for_init(project)
|
| 571 |
+
existing_run_records = _safe_get_runs_for_init(
|
| 572 |
+
project,
|
| 573 |
+
space_id,
|
| 574 |
+
server_base_url,
|
| 575 |
+
write_token_resolved,
|
| 576 |
+
resume,
|
| 577 |
+
remote_client=remote_client,
|
| 578 |
+
check_existing_for_never=name is not None,
|
| 579 |
+
)
|
| 580 |
+
existing_runs = [
|
| 581 |
+
r["name"] if isinstance(r, dict) else r for r in existing_run_records
|
| 582 |
+
]
|
| 583 |
+
|
| 584 |
+
existing_run = (
|
| 585 |
+
_safe_get_latest_run_for_init(
|
| 586 |
+
project,
|
| 587 |
+
name,
|
| 588 |
+
space_id=space_id,
|
| 589 |
+
server_base_url=server_base_url,
|
| 590 |
+
write_token=write_token_resolved,
|
| 591 |
+
remote_client=remote_client,
|
| 592 |
+
)
|
| 593 |
+
if name is not None
|
| 594 |
+
else None
|
| 595 |
+
)
|
| 596 |
+
resolved_run_id = None
|
| 597 |
+
|
| 598 |
+
if resume == "must":
|
| 599 |
+
if name is None:
|
| 600 |
+
raise ValueError("Must provide a run name when resume='must'")
|
| 601 |
+
if existing_run is None:
|
| 602 |
+
raise ValueError(f"Run '{name}' does not exist in project '{project}'")
|
| 603 |
+
resumed = True
|
| 604 |
+
resolved_run_id = existing_run["id"]
|
| 605 |
+
elif resume == "allow":
|
| 606 |
+
resumed = existing_run is not None
|
| 607 |
+
if resumed:
|
| 608 |
+
resolved_run_id = existing_run["id"]
|
| 609 |
+
elif resume == "never":
|
| 610 |
+
resumed = False
|
| 611 |
+
else:
|
| 612 |
+
raise ValueError("resume must be one of: 'must', 'allow', or 'never'")
|
| 613 |
+
|
| 614 |
+
initial_last_step = (
|
| 615 |
+
_safe_get_last_step_for_init(
|
| 616 |
+
project,
|
| 617 |
+
name,
|
| 618 |
+
space_id,
|
| 619 |
+
server_base_url,
|
| 620 |
+
write_token_resolved,
|
| 621 |
+
resumed,
|
| 622 |
+
run_id=resolved_run_id,
|
| 623 |
+
remote_client=remote_client,
|
| 624 |
+
)
|
| 625 |
+
if name is not None
|
| 626 |
+
else None
|
| 627 |
+
)
|
| 628 |
+
|
| 629 |
+
auto_log_cpu_detected = False
|
| 630 |
+
if auto_log_cpu is None:
|
| 631 |
+
auto_log_cpu_detected = cpu_available()
|
| 632 |
+
auto_log_cpu = auto_log_cpu_detected
|
| 633 |
+
|
| 634 |
+
auto_log_gpu_detected = False
|
| 635 |
+
nvidia_available = False
|
| 636 |
+
apple_available = False
|
| 637 |
+
if auto_log_gpu is None:
|
| 638 |
+
nvidia_available = gpu_available()
|
| 639 |
+
apple_available = apple_gpu_available()
|
| 640 |
+
auto_log_gpu_detected = nvidia_available or apple_available
|
| 641 |
+
auto_log_gpu = auto_log_gpu_detected
|
| 642 |
+
|
| 643 |
+
if project not in _projects_notified_auto_log_hw:
|
| 644 |
+
if nvidia_available:
|
| 645 |
+
print("* NVIDIA GPU detected, enabling automatic GPU metrics logging")
|
| 646 |
+
elif apple_available:
|
| 647 |
+
print(
|
| 648 |
+
"* Apple Silicon detected, enabling automatic GPU/system metrics logging"
|
| 649 |
+
)
|
| 650 |
+
if auto_log_cpu_detected:
|
| 651 |
+
print("* psutil detected, enabling automatic CPU/system metrics logging")
|
| 652 |
+
if auto_log_gpu_detected or auto_log_cpu_detected:
|
| 653 |
+
_projects_notified_auto_log_hw.add(project)
|
| 654 |
+
|
| 655 |
+
run = Run(
|
| 656 |
+
url=url,
|
| 657 |
+
project=project,
|
| 658 |
+
client=None,
|
| 659 |
+
name=name,
|
| 660 |
+
run_id=resolved_run_id,
|
| 661 |
+
group=group,
|
| 662 |
+
config=config,
|
| 663 |
+
space_id=space_id,
|
| 664 |
+
bucket_id=bucket_id,
|
| 665 |
+
server_base_url=server_base_url,
|
| 666 |
+
write_token=write_token_resolved,
|
| 667 |
+
existing_runs=existing_runs,
|
| 668 |
+
initial_last_step=initial_last_step,
|
| 669 |
+
auto_log_gpu=auto_log_gpu,
|
| 670 |
+
gpu_log_interval=gpu_log_interval,
|
| 671 |
+
auto_log_cpu=auto_log_cpu,
|
| 672 |
+
cpu_log_interval=cpu_log_interval,
|
| 673 |
+
webhook_url=webhook_url,
|
| 674 |
+
webhook_min_level=webhook_min_level,
|
| 675 |
+
)
|
| 676 |
+
|
| 677 |
+
if space_id is not None:
|
| 678 |
+
try:
|
| 679 |
+
SQLiteStorage.set_project_metadata(project, "space_id", space_id)
|
| 680 |
+
except Exception as e:
|
| 681 |
+
_emit_nonfatal_warning(
|
| 682 |
+
f"trackio.init() could not persist Space metadata for project '{project}': {e}. Logging will continue."
|
| 683 |
+
)
|
| 684 |
+
try:
|
| 685 |
+
if SQLiteStorage.has_pending_data(project):
|
| 686 |
+
run._has_local_buffer = True
|
| 687 |
+
except Exception as e:
|
| 688 |
+
_emit_nonfatal_warning(
|
| 689 |
+
f"trackio.init() could not inspect pending buffered data for project '{project}': {e}. Logging will continue."
|
| 690 |
+
)
|
| 691 |
+
|
| 692 |
+
global _atexit_registered
|
| 693 |
+
if not _atexit_registered:
|
| 694 |
+
atexit.register(_cleanup_current_run)
|
| 695 |
+
_atexit_registered = True
|
| 696 |
+
|
| 697 |
+
if resumed:
|
| 698 |
+
print(f"* Resumed existing run: {run.name}")
|
| 699 |
+
else:
|
| 700 |
+
print(f"* Created new run: {run.name}")
|
| 701 |
+
|
| 702 |
+
context_vars.current_run.set(run)
|
| 703 |
+
globals()["config"] = run.config
|
| 704 |
+
|
| 705 |
+
# NOTE: trackio.init() deliberately does NOT mutate any logbook that happens
|
| 706 |
+
# to live in the current directory. Auto-noting a dashboard cell here used to
|
| 707 |
+
# corrupt curated logbooks (e.g. injecting a stray cell / page). Cells are
|
| 708 |
+
# added explicitly via the `trackio logbook` CLI instead.
|
| 709 |
+
|
| 710 |
+
if _should_embed_local:
|
| 711 |
+
try:
|
| 712 |
+
show(project=project, open_browser=False, block_thread=False)
|
| 713 |
+
except Exception as e:
|
| 714 |
+
_emit_nonfatal_warning(
|
| 715 |
+
f"trackio.init() could not auto-launch the dashboard: {e}. Logging will continue."
|
| 716 |
+
)
|
| 717 |
+
|
| 718 |
+
return run
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
def log(metrics: dict, step: int | None = None) -> None:
|
| 722 |
+
"""
|
| 723 |
+
Logs metrics to the current run.
|
| 724 |
+
|
| 725 |
+
Args:
|
| 726 |
+
metrics (`dict`):
|
| 727 |
+
A dictionary of metrics to log.
|
| 728 |
+
step (`int`, *optional*):
|
| 729 |
+
The step number. If not provided, the step will be incremented
|
| 730 |
+
automatically.
|
| 731 |
+
"""
|
| 732 |
+
run = context_vars.current_run.get()
|
| 733 |
+
if run is None:
|
| 734 |
+
raise RuntimeError("Call trackio.init() before trackio.log().")
|
| 735 |
+
run.log(
|
| 736 |
+
metrics=metrics,
|
| 737 |
+
step=step,
|
| 738 |
+
)
|
| 739 |
+
|
| 740 |
+
|
| 741 |
+
def log_system(metrics: dict) -> None:
|
| 742 |
+
"""
|
| 743 |
+
Logs system metrics (GPU, etc.) to the current run using timestamps instead of steps.
|
| 744 |
+
|
| 745 |
+
Args:
|
| 746 |
+
metrics (`dict`):
|
| 747 |
+
A dictionary of system metrics to log.
|
| 748 |
+
"""
|
| 749 |
+
run = context_vars.current_run.get()
|
| 750 |
+
if run is None:
|
| 751 |
+
raise RuntimeError("Call trackio.init() before trackio.log_system().")
|
| 752 |
+
run.log_system(metrics=metrics)
|
| 753 |
+
|
| 754 |
+
|
| 755 |
+
def log_artifact(
|
| 756 |
+
artifact_or_path: Artifact | str | Path,
|
| 757 |
+
name: str | None = None,
|
| 758 |
+
type: str | None = None,
|
| 759 |
+
aliases: list[str] | None = None,
|
| 760 |
+
overwrite: bool = False,
|
| 761 |
+
) -> Artifact:
|
| 762 |
+
"""
|
| 763 |
+
Logs an artifact as an output of the current run.
|
| 764 |
+
|
| 765 |
+
Args:
|
| 766 |
+
artifact_or_path (`Artifact`, `str`, or `Path`):
|
| 767 |
+
The artifact to log (must have at least one file added via
|
| 768 |
+
`add_file` or `add_dir`), or a path to a file or directory to
|
| 769 |
+
log as a new artifact.
|
| 770 |
+
name (`str`, *optional*):
|
| 771 |
+
Artifact name when logging a path. Defaults to
|
| 772 |
+
`run-<run_id>-<basename>`. Must not be passed with an `Artifact`.
|
| 773 |
+
type (`str`, *optional*):
|
| 774 |
+
Artifact type when logging a path (e.g. `"model"`, `"dataset"`).
|
| 775 |
+
Defaults to `"unspecified"`. Must not be passed with an
|
| 776 |
+
`Artifact`.
|
| 777 |
+
aliases (`list[str]`, *optional*):
|
| 778 |
+
Aliases to rotate onto the resulting version, alongside `latest`
|
| 779 |
+
(assigned automatically whenever a new version is created). Your
|
| 780 |
+
aliases rotate onto the version even when identical content is
|
| 781 |
+
de-duplicated.
|
| 782 |
+
overwrite (`bool`, *optional*):
|
| 783 |
+
If `True`, removes every other version of this artifact after the
|
| 784 |
+
new content is logged. Blobs still used by other artifacts are
|
| 785 |
+
preserved, and version numbers continue increasing. Defaults to
|
| 786 |
+
`False`.
|
| 787 |
+
|
| 788 |
+
Returns:
|
| 789 |
+
The logged `Artifact` instance, hydrated with `version`, `aliases`,
|
| 790 |
+
`size`, `manifest`, and `project` set.
|
| 791 |
+
"""
|
| 792 |
+
run = context_vars.current_run.get()
|
| 793 |
+
if run is None:
|
| 794 |
+
raise RuntimeError("Call trackio.init() before trackio.log_artifact().")
|
| 795 |
+
return run.log_artifact(
|
| 796 |
+
artifact_or_path,
|
| 797 |
+
name=name,
|
| 798 |
+
type=type,
|
| 799 |
+
aliases=aliases,
|
| 800 |
+
overwrite=overwrite,
|
| 801 |
+
)
|
| 802 |
+
|
| 803 |
+
|
| 804 |
+
def use_artifact(
|
| 805 |
+
artifact_or_name: Artifact | str,
|
| 806 |
+
type: str | None = None,
|
| 807 |
+
) -> Artifact:
|
| 808 |
+
"""
|
| 809 |
+
Fetches an artifact and records it as an input to the current run.
|
| 810 |
+
|
| 811 |
+
Args:
|
| 812 |
+
artifact_or_name (`Artifact` or `str`):
|
| 813 |
+
An already-logged `Artifact`, or an artifact name. A bare name
|
| 814 |
+
(`"my-model"`) resolves to `:latest`; you can also pin a version
|
| 815 |
+
(`"my-model:v3"`) or resolve an alias (`"my-model:prod"`).
|
| 816 |
+
type (`str`, *optional*):
|
| 817 |
+
If given, checked against the stored artifact type, raising if it
|
| 818 |
+
does not match.
|
| 819 |
+
|
| 820 |
+
Returns:
|
| 821 |
+
The fetched `Artifact`, hydrated and ready to `download()`.
|
| 822 |
+
"""
|
| 823 |
+
run = context_vars.current_run.get()
|
| 824 |
+
if run is None:
|
| 825 |
+
raise RuntimeError("Call trackio.init() before trackio.use_artifact().")
|
| 826 |
+
return run.use_artifact(artifact_or_name, type=type)
|
| 827 |
+
|
| 828 |
+
|
| 829 |
+
def log_gpu(run: Run | None = None, device: int | None = None) -> dict:
|
| 830 |
+
"""
|
| 831 |
+
Log GPU metrics to the current or specified run as system metrics.
|
| 832 |
+
Automatically detects whether an NVIDIA or Apple GPU is available and calls
|
| 833 |
+
the appropriate logging method.
|
| 834 |
+
|
| 835 |
+
Args:
|
| 836 |
+
run: Optional Run instance. If None, uses current run from context.
|
| 837 |
+
device: CUDA device index to collect metrics from (NVIDIA GPUs only).
|
| 838 |
+
If None, collects from all GPUs visible to this process.
|
| 839 |
+
This parameter is ignored for Apple GPUs.
|
| 840 |
+
|
| 841 |
+
Returns:
|
| 842 |
+
dict: The GPU metrics that were logged.
|
| 843 |
+
|
| 844 |
+
Example:
|
| 845 |
+
```python
|
| 846 |
+
import trackio
|
| 847 |
+
|
| 848 |
+
run = trackio.init(project="my-project")
|
| 849 |
+
trackio.log({"loss": 0.5})
|
| 850 |
+
trackio.log_gpu()
|
| 851 |
+
trackio.log_gpu(device=0)
|
| 852 |
+
```
|
| 853 |
+
"""
|
| 854 |
+
if run is None:
|
| 855 |
+
run = context_vars.current_run.get()
|
| 856 |
+
if run is None:
|
| 857 |
+
raise RuntimeError("Call trackio.init() before trackio.log_gpu().")
|
| 858 |
+
|
| 859 |
+
if gpu_available():
|
| 860 |
+
return _log_nvidia_gpu(run=run, device=device)
|
| 861 |
+
elif apple_gpu_available():
|
| 862 |
+
return _log_apple_gpu(run=run)
|
| 863 |
+
else:
|
| 864 |
+
_emit_nonfatal_warning(
|
| 865 |
+
"No GPU detected. Install nvidia-ml-py for NVIDIA GPU support "
|
| 866 |
+
"or psutil for Apple Silicon support."
|
| 867 |
+
)
|
| 868 |
+
return {}
|
| 869 |
+
|
| 870 |
+
|
| 871 |
+
def log_cpu(run: Run | None = None) -> dict:
|
| 872 |
+
"""
|
| 873 |
+
Log CPU, RAM, disk, network, and sensor metrics to the current or specified run
|
| 874 |
+
as system metrics.
|
| 875 |
+
|
| 876 |
+
Args:
|
| 877 |
+
run: Optional Run instance. If None, uses current run from context.
|
| 878 |
+
|
| 879 |
+
Returns:
|
| 880 |
+
dict: The CPU and system metrics that were logged.
|
| 881 |
+
|
| 882 |
+
Example:
|
| 883 |
+
```python
|
| 884 |
+
import trackio
|
| 885 |
+
|
| 886 |
+
run = trackio.init(project="my-project")
|
| 887 |
+
trackio.log({"loss": 0.5})
|
| 888 |
+
trackio.log_cpu()
|
| 889 |
+
```
|
| 890 |
+
"""
|
| 891 |
+
if run is None:
|
| 892 |
+
run = context_vars.current_run.get()
|
| 893 |
+
if run is None:
|
| 894 |
+
raise RuntimeError("Call trackio.init() before trackio.log_cpu().")
|
| 895 |
+
|
| 896 |
+
return _log_cpu(run=run)
|
| 897 |
+
|
| 898 |
+
|
| 899 |
+
def finish():
|
| 900 |
+
"""
|
| 901 |
+
Finishes the current run.
|
| 902 |
+
"""
|
| 903 |
+
run = context_vars.current_run.get()
|
| 904 |
+
if run is None:
|
| 905 |
+
raise RuntimeError("Call trackio.init() before trackio.finish().")
|
| 906 |
+
try:
|
| 907 |
+
run.finish()
|
| 908 |
+
finally:
|
| 909 |
+
context_vars.current_run.set(None)
|
| 910 |
+
|
| 911 |
+
|
| 912 |
+
def alert(
|
| 913 |
+
title: str,
|
| 914 |
+
text: str | None = None,
|
| 915 |
+
level: AlertLevel = AlertLevel.WARN,
|
| 916 |
+
webhook_url: str | None = None,
|
| 917 |
+
) -> None:
|
| 918 |
+
"""
|
| 919 |
+
Fires an alert immediately on the current run. The alert is printed to the
|
| 920 |
+
terminal, stored in the database, and displayed in the dashboard. If a
|
| 921 |
+
webhook URL is configured (via `trackio.init()`, the `TRACKIO_WEBHOOK_URL`
|
| 922 |
+
environment variable, or the `webhook_url` parameter here), the alert is
|
| 923 |
+
also POSTed to that URL.
|
| 924 |
+
|
| 925 |
+
Args:
|
| 926 |
+
title (`str`):
|
| 927 |
+
A short title for the alert.
|
| 928 |
+
text (`str`, *optional*):
|
| 929 |
+
A longer description with details about the alert.
|
| 930 |
+
level (`AlertLevel`, *optional*, defaults to `AlertLevel.WARN`):
|
| 931 |
+
The severity level. One of `AlertLevel.INFO`, `AlertLevel.WARN`,
|
| 932 |
+
or `AlertLevel.ERROR`.
|
| 933 |
+
webhook_url (`str`, *optional*):
|
| 934 |
+
A webhook URL to send this specific alert to. Overrides any
|
| 935 |
+
URL set in `trackio.init()` or the `TRACKIO_WEBHOOK_URL`
|
| 936 |
+
environment variable. Supports Slack and Discord webhook
|
| 937 |
+
URLs natively.
|
| 938 |
+
"""
|
| 939 |
+
run = context_vars.current_run.get()
|
| 940 |
+
if run is None:
|
| 941 |
+
raise RuntimeError("Call trackio.init() before trackio.alert().")
|
| 942 |
+
run.alert(title=title, text=text, level=level, webhook_url=webhook_url)
|
| 943 |
+
|
| 944 |
+
|
| 945 |
+
def delete_project(project: str, force: bool = False) -> bool:
|
| 946 |
+
"""
|
| 947 |
+
Deletes a project by removing its local SQLite database.
|
| 948 |
+
|
| 949 |
+
Args:
|
| 950 |
+
project (`str`):
|
| 951 |
+
The name of the project to delete.
|
| 952 |
+
force (`bool`, *optional*, defaults to `False`):
|
| 953 |
+
If `True`, deletes the project without prompting for confirmation.
|
| 954 |
+
If `False`, prompts the user to confirm before deleting.
|
| 955 |
+
|
| 956 |
+
Returns:
|
| 957 |
+
`bool`: `True` if the project was deleted, `False` otherwise.
|
| 958 |
+
"""
|
| 959 |
+
db_path = SQLiteStorage.get_project_db_path(project)
|
| 960 |
+
|
| 961 |
+
if not db_path.exists():
|
| 962 |
+
print(f"* Project '{project}' does not exist.")
|
| 963 |
+
return False
|
| 964 |
+
|
| 965 |
+
if not force:
|
| 966 |
+
response = input(
|
| 967 |
+
f"Are you sure you want to delete project '{project}'? "
|
| 968 |
+
f"This will permanently delete all runs and metrics. (y/N): "
|
| 969 |
+
)
|
| 970 |
+
if response.lower() not in ["y", "yes"]:
|
| 971 |
+
print("* Deletion cancelled.")
|
| 972 |
+
return False
|
| 973 |
+
|
| 974 |
+
try:
|
| 975 |
+
db_path.unlink()
|
| 976 |
+
|
| 977 |
+
for suffix in ("-wal", "-shm"):
|
| 978 |
+
sidecar = Path(str(db_path) + suffix)
|
| 979 |
+
if sidecar.exists():
|
| 980 |
+
sidecar.unlink()
|
| 981 |
+
|
| 982 |
+
for parquet_path in SQLiteStorage._project_parquet_paths(db_path):
|
| 983 |
+
if parquet_path.exists():
|
| 984 |
+
parquet_path.unlink()
|
| 985 |
+
|
| 986 |
+
for asset_dir in (
|
| 987 |
+
utils.project_artifacts_dir(project),
|
| 988 |
+
utils.project_media_dir(project),
|
| 989 |
+
):
|
| 990 |
+
if asset_dir.exists():
|
| 991 |
+
shutil.rmtree(asset_dir, ignore_errors=True)
|
| 992 |
+
|
| 993 |
+
print(f"* Project '{project}' has been deleted.")
|
| 994 |
+
return True
|
| 995 |
+
except Exception as e:
|
| 996 |
+
print(f"* Error deleting project '{project}': {e}")
|
| 997 |
+
return False
|
| 998 |
+
|
| 999 |
+
|
| 1000 |
+
def save(
|
| 1001 |
+
glob_str: str | Path,
|
| 1002 |
+
project: str | None = None,
|
| 1003 |
+
) -> str:
|
| 1004 |
+
"""
|
| 1005 |
+
Saves files to a project (not linked to a specific run). If Trackio is running
|
| 1006 |
+
locally, the file(s) will be copied to the project's files directory. If Trackio is
|
| 1007 |
+
running in a Space, the file(s) will be uploaded to the Space's files directory.
|
| 1008 |
+
|
| 1009 |
+
Args:
|
| 1010 |
+
glob_str (`str` or `Path`):
|
| 1011 |
+
The file path or glob pattern to save. Can be a single file or a pattern
|
| 1012 |
+
matching multiple files (e.g., `"*.py"`, `"models/**/*.pth"`).
|
| 1013 |
+
project (`str`, *optional*):
|
| 1014 |
+
The name of the project to save files to. If not provided, uses the current
|
| 1015 |
+
project from `trackio.init()`. If no project is initialized, raises an
|
| 1016 |
+
error.
|
| 1017 |
+
|
| 1018 |
+
Returns:
|
| 1019 |
+
`str`: The path where the file(s) were saved (project's files directory).
|
| 1020 |
+
|
| 1021 |
+
Example:
|
| 1022 |
+
```python
|
| 1023 |
+
import trackio
|
| 1024 |
+
|
| 1025 |
+
trackio.init(project="my-project")
|
| 1026 |
+
trackio.save("config.yaml")
|
| 1027 |
+
trackio.save("models/*.pth")
|
| 1028 |
+
```
|
| 1029 |
+
"""
|
| 1030 |
+
if project is None:
|
| 1031 |
+
project = context_vars.current_project.get()
|
| 1032 |
+
if project is None:
|
| 1033 |
+
raise RuntimeError(
|
| 1034 |
+
"No project specified. Either call trackio.init() first or provide a "
|
| 1035 |
+
"project parameter to trackio.save()."
|
| 1036 |
+
)
|
| 1037 |
+
|
| 1038 |
+
glob_str = Path(glob_str)
|
| 1039 |
+
base_path = Path.cwd().resolve()
|
| 1040 |
+
|
| 1041 |
+
matched_files = []
|
| 1042 |
+
if glob_str.is_file():
|
| 1043 |
+
matched_files = [glob_str.resolve()]
|
| 1044 |
+
else:
|
| 1045 |
+
pattern = str(glob_str)
|
| 1046 |
+
if not glob_str.is_absolute():
|
| 1047 |
+
pattern = str((Path.cwd() / glob_str).resolve())
|
| 1048 |
+
matched_files = [
|
| 1049 |
+
Path(f).resolve()
|
| 1050 |
+
for f in glob.glob(pattern, recursive=True)
|
| 1051 |
+
if Path(f).is_file()
|
| 1052 |
+
]
|
| 1053 |
+
|
| 1054 |
+
if not matched_files:
|
| 1055 |
+
raise ValueError(f"No files found matching pattern: {glob_str}")
|
| 1056 |
+
|
| 1057 |
+
current_run = context_vars.current_run.get()
|
| 1058 |
+
is_local = (
|
| 1059 |
+
current_run._is_local
|
| 1060 |
+
if current_run is not None
|
| 1061 |
+
else (
|
| 1062 |
+
context_vars.current_space_id.get() is None
|
| 1063 |
+
and context_vars.current_server.get() is None
|
| 1064 |
+
)
|
| 1065 |
+
)
|
| 1066 |
+
|
| 1067 |
+
if is_local:
|
| 1068 |
+
for file_path in matched_files:
|
| 1069 |
+
try:
|
| 1070 |
+
relative_to_base = file_path.relative_to(base_path)
|
| 1071 |
+
except ValueError:
|
| 1072 |
+
relative_to_base = Path(file_path.name)
|
| 1073 |
+
|
| 1074 |
+
if current_run is not None:
|
| 1075 |
+
current_run._queue_upload(
|
| 1076 |
+
file_path,
|
| 1077 |
+
step=None,
|
| 1078 |
+
relative_path=str(relative_to_base.parent),
|
| 1079 |
+
use_run_name=False,
|
| 1080 |
+
)
|
| 1081 |
+
else:
|
| 1082 |
+
media_path = get_project_media_path(
|
| 1083 |
+
project=project,
|
| 1084 |
+
run=None,
|
| 1085 |
+
step=None,
|
| 1086 |
+
relative_path=str(relative_to_base),
|
| 1087 |
+
)
|
| 1088 |
+
shutil.copy(str(file_path), str(media_path))
|
| 1089 |
+
else:
|
| 1090 |
+
url = context_vars.current_server.get()
|
| 1091 |
+
|
| 1092 |
+
upload_entries = []
|
| 1093 |
+
for file_path in matched_files:
|
| 1094 |
+
try:
|
| 1095 |
+
relative_to_base = file_path.relative_to(base_path)
|
| 1096 |
+
except ValueError:
|
| 1097 |
+
relative_to_base = Path(file_path.name)
|
| 1098 |
+
|
| 1099 |
+
if current_run is not None:
|
| 1100 |
+
current_run._queue_upload(
|
| 1101 |
+
file_path,
|
| 1102 |
+
step=None,
|
| 1103 |
+
relative_path=str(relative_to_base.parent),
|
| 1104 |
+
use_run_name=False,
|
| 1105 |
+
)
|
| 1106 |
+
else:
|
| 1107 |
+
upload_entry: UploadEntry = {
|
| 1108 |
+
"project": project,
|
| 1109 |
+
"run": None,
|
| 1110 |
+
"step": None,
|
| 1111 |
+
"relative_path": str(relative_to_base),
|
| 1112 |
+
"uploaded_file": handle_file(file_path),
|
| 1113 |
+
}
|
| 1114 |
+
upload_entries.append(upload_entry)
|
| 1115 |
+
|
| 1116 |
+
if upload_entries:
|
| 1117 |
+
if url is None:
|
| 1118 |
+
raise RuntimeError(
|
| 1119 |
+
"No server available. Call trackio.init() before trackio.save() to start the server."
|
| 1120 |
+
)
|
| 1121 |
+
|
| 1122 |
+
try:
|
| 1123 |
+
wt = context_vars.current_server_write_token.get()
|
| 1124 |
+
if wt is not None:
|
| 1125 |
+
client = RemoteClient(
|
| 1126 |
+
url,
|
| 1127 |
+
hf_token=None,
|
| 1128 |
+
write_token=wt,
|
| 1129 |
+
httpx_kwargs={"timeout": 90},
|
| 1130 |
+
)
|
| 1131 |
+
else:
|
| 1132 |
+
client = RemoteClient(
|
| 1133 |
+
url,
|
| 1134 |
+
hf_token=huggingface_hub.utils.get_token(),
|
| 1135 |
+
httpx_kwargs={"timeout": 90},
|
| 1136 |
+
)
|
| 1137 |
+
client.predict(
|
| 1138 |
+
api_name="/bulk_upload_media",
|
| 1139 |
+
uploads=upload_entries,
|
| 1140 |
+
hf_token=huggingface_hub.utils.get_token() if wt is None else None,
|
| 1141 |
+
)
|
| 1142 |
+
except Exception as e:
|
| 1143 |
+
_emit_nonfatal_warning(
|
| 1144 |
+
f"Failed to upload files: {e}. "
|
| 1145 |
+
"Files may not be available in the dashboard."
|
| 1146 |
+
)
|
| 1147 |
+
|
| 1148 |
+
return str(utils.project_media_dir(project) / "files")
|
| 1149 |
+
|
| 1150 |
+
|
| 1151 |
+
def show(
|
| 1152 |
+
project: str | None = None,
|
| 1153 |
+
*,
|
| 1154 |
+
dashboard: str | None = None,
|
| 1155 |
+
theme: Any = None,
|
| 1156 |
+
mcp_server: bool | None = None,
|
| 1157 |
+
footer: bool = True,
|
| 1158 |
+
color_palette: list[str] | None = None,
|
| 1159 |
+
open_browser: bool = True,
|
| 1160 |
+
block_thread: bool | None = None,
|
| 1161 |
+
host: str | None = None,
|
| 1162 |
+
share: bool | None = None,
|
| 1163 |
+
server_port: int | None = None,
|
| 1164 |
+
frontend_dir: str | Path | None = None,
|
| 1165 |
+
):
|
| 1166 |
+
"""
|
| 1167 |
+
Launches the Trackio dashboard.
|
| 1168 |
+
|
| 1169 |
+
Args:
|
| 1170 |
+
project (`str`, *optional*):
|
| 1171 |
+
The name of the project whose runs to show. If not provided, all projects
|
| 1172 |
+
will be shown and the user can select one.
|
| 1173 |
+
dashboard (`str`, *optional*):
|
| 1174 |
+
Set to `"registry"` to show the standalone registry dashboard. The
|
| 1175 |
+
default is the project dashboard.
|
| 1176 |
+
theme (`Any`, *optional*):
|
| 1177 |
+
Ignored. Kept for backward compatibility; Trackio no longer uses Gradio themes.
|
| 1178 |
+
mcp_server (`bool`, *optional*):
|
| 1179 |
+
If `True`, the dashboard exposes an MCP server at `/mcp` when the optional
|
| 1180 |
+
`trackio[mcp]` dependency is installed. If `None` (default), the
|
| 1181 |
+
`GRADIO_MCP_SERVER` environment variable is used (e.g. on Spaces).
|
| 1182 |
+
footer (`bool`, *optional*, defaults to `True`):
|
| 1183 |
+
Whether to include `footer=false` in the write-token URL when `False`.
|
| 1184 |
+
This can also be controlled via the `footer` query parameter in the URL.
|
| 1185 |
+
color_palette (`list[str]`, *optional*):
|
| 1186 |
+
A list of hex color codes to use for plot lines. If not provided, the
|
| 1187 |
+
`TRACKIO_COLOR_PALETTE` environment variable will be used (comma-separated
|
| 1188 |
+
hex codes), or if that is not set, the default color palette will be used.
|
| 1189 |
+
Example: `['#FF0000', '#00FF00', '#0000FF']`
|
| 1190 |
+
open_browser (`bool`, *optional*, defaults to `True`):
|
| 1191 |
+
If `True` and not in a notebook, a new browser tab will be opened with the
|
| 1192 |
+
dashboard. If `False`, the browser will not be opened.
|
| 1193 |
+
block_thread (`bool`, *optional*):
|
| 1194 |
+
If `True`, the main thread will be blocked until the dashboard is closed.
|
| 1195 |
+
If `None` (default behavior), then the main thread will not be blocked if the
|
| 1196 |
+
dashboard is launched in a notebook, otherwise the main thread will be blocked.
|
| 1197 |
+
host (`str`, *optional*):
|
| 1198 |
+
The host to bind the server to. If not provided, defaults to `'127.0.0.1'`
|
| 1199 |
+
(localhost only). Set to `'0.0.0.0'` to allow remote access.
|
| 1200 |
+
share (`bool`, *optional*):
|
| 1201 |
+
If `True`, creates a temporary public URL (Gradio-compatible tunnel). On Colab
|
| 1202 |
+
or hosted notebooks, defaults to `True` unless overridden.
|
| 1203 |
+
server_port (`int`, *optional*):
|
| 1204 |
+
Port to bind. If not set, scans from `GRADIO_SERVER_PORT` (default 7860).
|
| 1205 |
+
frontend_dir (`str | Path`, *optional*):
|
| 1206 |
+
Directory containing a custom static frontend. Must contain `index.html`.
|
| 1207 |
+
If not provided, Trackio checks `TRACKIO_FRONTEND_DIR`, then the persistent
|
| 1208 |
+
Trackio config, then the bundled frontend. If an explicit `frontend_dir`
|
| 1209 |
+
points to a missing or empty directory, Trackio copies in the starter
|
| 1210 |
+
template and serves that directory.
|
| 1211 |
+
|
| 1212 |
+
Returns:
|
| 1213 |
+
`app`: The dashboard handle (`.close()` stops the server).
|
| 1214 |
+
`url`: The local URL of the dashboard.
|
| 1215 |
+
`share_url`: The public share URL, if any.
|
| 1216 |
+
`full_url`: The full URL including the write token (share URL when sharing, else local).
|
| 1217 |
+
"""
|
| 1218 |
+
if dashboard not in (None, "registry"):
|
| 1219 |
+
raise ValueError("dashboard must be None or 'registry'.")
|
| 1220 |
+
if dashboard == "registry" and project is not None:
|
| 1221 |
+
raise ValueError("The registry dashboard does not accept a project.")
|
| 1222 |
+
|
| 1223 |
+
if theme is not None and theme != "default":
|
| 1224 |
+
warnings.warn(
|
| 1225 |
+
"The theme argument is ignored; Trackio no longer depends on Gradio themes.",
|
| 1226 |
+
UserWarning,
|
| 1227 |
+
stacklevel=2,
|
| 1228 |
+
)
|
| 1229 |
+
|
| 1230 |
+
if color_palette is not None:
|
| 1231 |
+
os.environ["TRACKIO_COLOR_PALETTE"] = ",".join(color_palette)
|
| 1232 |
+
|
| 1233 |
+
_mcp_server = (
|
| 1234 |
+
mcp_server
|
| 1235 |
+
if mcp_server is not None
|
| 1236 |
+
else os.environ.get("GRADIO_MCP_SERVER", "False") == "True"
|
| 1237 |
+
)
|
| 1238 |
+
|
| 1239 |
+
resolved_frontend = resolve_frontend_dir(frontend_dir, announce=True)
|
| 1240 |
+
starlette_app, wt = build_starlette_app_only(
|
| 1241 |
+
mcp_server=_mcp_server,
|
| 1242 |
+
frontend_dir=str(resolved_frontend.path),
|
| 1243 |
+
)
|
| 1244 |
+
local_url, share_url, _local_api_url, uv_server = launch_trackio_dashboard(
|
| 1245 |
+
starlette_app,
|
| 1246 |
+
server_name=host,
|
| 1247 |
+
server_port=server_port,
|
| 1248 |
+
share=share,
|
| 1249 |
+
mcp_server=_mcp_server,
|
| 1250 |
+
quiet=True,
|
| 1251 |
+
)
|
| 1252 |
+
server = TrackioDashboardApp(starlette_app, uv_server, wt)
|
| 1253 |
+
|
| 1254 |
+
base_root = (share_url or local_url).rstrip("/")
|
| 1255 |
+
base_url = base_root + "/"
|
| 1256 |
+
dashboard_url = base_url if dashboard is None else f"{base_root}/registry"
|
| 1257 |
+
if dashboard is None and project:
|
| 1258 |
+
dashboard_url += f"?project={project}"
|
| 1259 |
+
full_url_base = base_root if dashboard is None else f"{base_root}/registry"
|
| 1260 |
+
full_url = utils.get_full_url(
|
| 1261 |
+
full_url_base,
|
| 1262 |
+
project=project,
|
| 1263 |
+
write_token=wt,
|
| 1264 |
+
footer=footer,
|
| 1265 |
+
)
|
| 1266 |
+
|
| 1267 |
+
if not utils.is_in_notebook():
|
| 1268 |
+
label = "Trackio registry UI" if dashboard == "registry" else "Trackio UI"
|
| 1269 |
+
print(f"\033[1m\033[38;5;208m* {label} launched at: {dashboard_url}\033[0m")
|
| 1270 |
+
utils.print_write_token_instructions(full_url)
|
| 1271 |
+
if open_browser:
|
| 1272 |
+
webbrowser.open(full_url)
|
| 1273 |
+
block_thread = block_thread if block_thread is not None else True
|
| 1274 |
+
else:
|
| 1275 |
+
utils.embed_url_in_notebook(full_url)
|
| 1276 |
+
block_thread = block_thread if block_thread is not None else False
|
| 1277 |
+
|
| 1278 |
+
if block_thread:
|
| 1279 |
+
utils.block_main_thread_until_keyboard_interrupt()
|
| 1280 |
+
return _TupleNoPrint((server, local_url, share_url, full_url))
|
trackio/_unsupported.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Placeholders for W&B feature classes that Trackio does not yet implement."""
|
| 2 |
+
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class _UnsupportedFeature:
|
| 7 |
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
| 8 |
+
raise NotImplementedError(
|
| 9 |
+
f"trackio.{type(self).__name__} is not implemented yet. "
|
| 10 |
+
"If you need this feature, please open a GitHub issue in the Trackio "
|
| 11 |
+
"repository: https://github.com/gradio-app/trackio/issues/new"
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class ArtifactTTL(_UnsupportedFeature):
|
| 16 |
+
"""Placeholder for artifact time-to-live policies."""
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class Classes(_UnsupportedFeature):
|
| 20 |
+
"""Placeholder for class labels used in media annotations."""
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class Config(_UnsupportedFeature):
|
| 24 |
+
"""Placeholder for the W&B configuration container."""
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class EvalTable(_UnsupportedFeature):
|
| 28 |
+
"""Placeholder for evaluation tables."""
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class Graph(_UnsupportedFeature):
|
| 32 |
+
"""Placeholder for model graphs."""
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class JoinedTable(_UnsupportedFeature):
|
| 36 |
+
"""Placeholder for joining tables by a shared key."""
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class Molecule(_UnsupportedFeature):
|
| 40 |
+
"""Placeholder for molecular structure visualizations."""
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class Plotly(_UnsupportedFeature):
|
| 44 |
+
"""Placeholder for Plotly visualizations."""
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class Settings(_UnsupportedFeature):
|
| 48 |
+
"""Placeholder for W&B SDK settings."""
|
trackio/_vendor/__init__.py
ADDED
|
File without changes
|
trackio/_vendor/gradio_exceptions.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
class ChecksumMismatchError(Exception):
|
| 2 |
+
pass
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class ShareCertificateWriteError(Exception):
|
| 6 |
+
pass
|
trackio/_vendor/networking.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import time
|
| 5 |
+
import warnings
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from urllib.parse import urlparse, urlunparse
|
| 8 |
+
|
| 9 |
+
import httpx
|
| 10 |
+
|
| 11 |
+
from trackio._vendor.gradio_exceptions import ShareCertificateWriteError
|
| 12 |
+
from trackio._vendor.tunneling import CERTIFICATE_PATH, Tunnel
|
| 13 |
+
|
| 14 |
+
GRADIO_API_SERVER = "https://api.gradio.app/v3/tunnel-request"
|
| 15 |
+
GRADIO_SHARE_SERVER_ADDRESS = os.getenv("GRADIO_SHARE_SERVER_ADDRESS")
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def setup_tunnel(
|
| 19 |
+
local_host: str,
|
| 20 |
+
local_port: int,
|
| 21 |
+
share_token: str,
|
| 22 |
+
share_server_address: str | None,
|
| 23 |
+
share_server_tls_certificate: str | None,
|
| 24 |
+
) -> str:
|
| 25 |
+
share_server_address = (
|
| 26 |
+
GRADIO_SHARE_SERVER_ADDRESS
|
| 27 |
+
if share_server_address is None
|
| 28 |
+
else share_server_address
|
| 29 |
+
)
|
| 30 |
+
if share_server_address is None:
|
| 31 |
+
try:
|
| 32 |
+
response = httpx.get(GRADIO_API_SERVER, timeout=30)
|
| 33 |
+
payload = response.json()[0]
|
| 34 |
+
remote_host, remote_port = payload["host"], int(payload["port"])
|
| 35 |
+
certificate = payload["root_ca"]
|
| 36 |
+
except Exception as e:
|
| 37 |
+
raise RuntimeError(
|
| 38 |
+
"Could not get share link from Gradio API Server."
|
| 39 |
+
) from e
|
| 40 |
+
try:
|
| 41 |
+
Path(CERTIFICATE_PATH).parent.mkdir(parents=True, exist_ok=True)
|
| 42 |
+
with open(CERTIFICATE_PATH, "w") as f:
|
| 43 |
+
f.write(certificate)
|
| 44 |
+
except Exception as e:
|
| 45 |
+
raise ShareCertificateWriteError(
|
| 46 |
+
f"{e}. This can happen if the current working directory is read-only."
|
| 47 |
+
) from e
|
| 48 |
+
share_server_tls_certificate = CERTIFICATE_PATH
|
| 49 |
+
|
| 50 |
+
else:
|
| 51 |
+
remote_host, remote_port = share_server_address.split(":")
|
| 52 |
+
remote_port = int(remote_port)
|
| 53 |
+
tunnel = Tunnel(
|
| 54 |
+
remote_host,
|
| 55 |
+
remote_port,
|
| 56 |
+
local_host,
|
| 57 |
+
local_port,
|
| 58 |
+
share_token,
|
| 59 |
+
share_server_tls_certificate,
|
| 60 |
+
)
|
| 61 |
+
address = tunnel.start_tunnel()
|
| 62 |
+
return address
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def url_ok(url: str) -> bool:
|
| 66 |
+
try:
|
| 67 |
+
for _ in range(5):
|
| 68 |
+
with warnings.catch_warnings():
|
| 69 |
+
warnings.filterwarnings("ignore")
|
| 70 |
+
r = httpx.head(url, timeout=3, verify=False)
|
| 71 |
+
if r.status_code in (200, 401, 302, 303, 307):
|
| 72 |
+
return True
|
| 73 |
+
time.sleep(0.500)
|
| 74 |
+
except (ConnectionError, httpx.ConnectError, httpx.TimeoutException):
|
| 75 |
+
return False
|
| 76 |
+
return False
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def normalize_share_url(share_url: str, share_server_protocol: str) -> str:
|
| 80 |
+
parsed_url = urlparse(share_url)
|
| 81 |
+
return urlunparse((share_server_protocol,) + parsed_url[1:])
|
trackio/_vendor/tunneling.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import atexit
|
| 2 |
+
import hashlib
|
| 3 |
+
import os
|
| 4 |
+
import platform
|
| 5 |
+
import re
|
| 6 |
+
import stat
|
| 7 |
+
import subprocess
|
| 8 |
+
import sys
|
| 9 |
+
import time
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
import httpx
|
| 13 |
+
from huggingface_hub.constants import HF_HOME
|
| 14 |
+
|
| 15 |
+
from trackio._vendor.gradio_exceptions import ChecksumMismatchError
|
| 16 |
+
|
| 17 |
+
VERSION = "0.3"
|
| 18 |
+
CURRENT_TUNNELS: list["Tunnel"] = []
|
| 19 |
+
|
| 20 |
+
machine = platform.machine()
|
| 21 |
+
if machine == "x86_64":
|
| 22 |
+
machine = "amd64"
|
| 23 |
+
elif machine == "aarch64":
|
| 24 |
+
machine = "arm64"
|
| 25 |
+
|
| 26 |
+
BINARY_REMOTE_NAME = f"frpc_{platform.system().lower()}_{machine.lower()}"
|
| 27 |
+
EXTENSION = ".exe" if os.name == "nt" else ""
|
| 28 |
+
BINARY_URL = f"https://cdn-media.huggingface.co/frpc-gradio-{VERSION}/{BINARY_REMOTE_NAME}{EXTENSION}"
|
| 29 |
+
|
| 30 |
+
CHECKSUMS = {
|
| 31 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_windows_amd64.exe": "14bc0ea470be5d67d79a07412bd21de8a0a179c6ac1116d7764f68e942dc9ceb",
|
| 32 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_amd64": "c791d1f047b41ff5885772fc4bf20b797c6059bbd82abb9e31de15e55d6a57c4",
|
| 33 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_arm64": "823ced25104de6dc3c9f4798dbb43f20e681207279e6ab89c40e2176ccbf70cd",
|
| 34 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_darwin_amd64": "930f8face3365810ce16689da81b7d1941fda4466225a7bbcbced9a2916a6e15",
|
| 35 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_darwin_arm64": "dfac50c690aca459ed5158fad8bfbe99f9282baf4166cf7c410a6673fbc1f327",
|
| 36 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_arm": "4b563beb2e36c448cc688174e20b53af38dc1ff2b5e362d4ddd1401f2affbfb7",
|
| 37 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_freebsd_386": "cb0a56c764ecf96dd54ed601d240c564f060ee4e58202d65ffca17c1a51ce19c",
|
| 38 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_freebsd_amd64": "516d9e6903513869a011ddcd1ec206167ad1eb5dd6640d21057acc258edecbbb",
|
| 39 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_386": "4c2f2a48cd71571498c0ac8a4d42a055f22cb7f14b4b5a2b0d584220fd60a283",
|
| 40 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mips": "b309ecd594d4f0f7f33e556a80d4b67aef9319c00a8334648a618e56b23cb9e0",
|
| 41 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mips64": "0372ef5505baa6f3b64c6295a86541b24b7b0dbe4ef28b344992e21f47624b7b",
|
| 42 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_riscv64": "1658eed7e8c14ea76e1d95749d58441ce24147c3d559381832c725c29cfc3df3",
|
| 43 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mipsle": "a2aaba16961d3372b79bd7a28976fcd0f0bbaebc2b50d5a7a71af2240747960f",
|
| 44 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_windows_386.exe": "721b90550195a83e15f2176d8f85a48d5a25822757cb872e9723d4bccc4e5bb6",
|
| 45 |
+
"https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_mips64le": "796481edd609f31962b45cc0ab4c9798d040205ae3bf354ed1b72fb432d796b8",
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
CHUNK_SIZE = 128
|
| 49 |
+
|
| 50 |
+
BINARY_FILENAME = f"{BINARY_REMOTE_NAME}_v{VERSION}"
|
| 51 |
+
BINARY_FOLDER = Path(HF_HOME) / "trackio" / "frpc"
|
| 52 |
+
BINARY_PATH = str(BINARY_FOLDER / BINARY_FILENAME)
|
| 53 |
+
|
| 54 |
+
TUNNEL_TIMEOUT_SECONDS = 30
|
| 55 |
+
TUNNEL_ERROR_MESSAGE = (
|
| 56 |
+
"Could not create share URL. "
|
| 57 |
+
"Please check the appended log from frpc for more information:"
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
CERTIFICATE_PATH = ".trackio/certificate.pem"
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class Tunnel:
|
| 64 |
+
def __init__(
|
| 65 |
+
self,
|
| 66 |
+
remote_host: str,
|
| 67 |
+
remote_port: int,
|
| 68 |
+
local_host: str,
|
| 69 |
+
local_port: int,
|
| 70 |
+
share_token: str,
|
| 71 |
+
share_server_tls_certificate: str | None,
|
| 72 |
+
):
|
| 73 |
+
self.proc = None
|
| 74 |
+
self.url = None
|
| 75 |
+
self.remote_host = remote_host
|
| 76 |
+
self.remote_port = remote_port
|
| 77 |
+
self.local_host = local_host
|
| 78 |
+
self.local_port = local_port
|
| 79 |
+
self.share_token = share_token
|
| 80 |
+
self.share_server_tls_certificate = share_server_tls_certificate
|
| 81 |
+
|
| 82 |
+
@staticmethod
|
| 83 |
+
def download_binary():
|
| 84 |
+
if not Path(BINARY_PATH).exists():
|
| 85 |
+
Path(BINARY_FOLDER).mkdir(parents=True, exist_ok=True)
|
| 86 |
+
resp = httpx.get(BINARY_URL, timeout=30)
|
| 87 |
+
|
| 88 |
+
if resp.status_code == 403:
|
| 89 |
+
raise OSError(
|
| 90 |
+
f"Cannot set up a share link as this platform is incompatible. Please "
|
| 91 |
+
f"create a GitHub issue with information about your platform: {platform.uname()}"
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
resp.raise_for_status()
|
| 95 |
+
|
| 96 |
+
with open(BINARY_PATH, "wb") as file:
|
| 97 |
+
file.write(resp.content)
|
| 98 |
+
st = os.stat(BINARY_PATH)
|
| 99 |
+
os.chmod(BINARY_PATH, st.st_mode | stat.S_IEXEC)
|
| 100 |
+
|
| 101 |
+
if BINARY_URL in CHECKSUMS:
|
| 102 |
+
sha = hashlib.sha256()
|
| 103 |
+
with open(BINARY_PATH, "rb") as f:
|
| 104 |
+
for chunk in iter(lambda: f.read(CHUNK_SIZE * sha.block_size), b""):
|
| 105 |
+
sha.update(chunk)
|
| 106 |
+
calculated_hash = sha.hexdigest()
|
| 107 |
+
|
| 108 |
+
if calculated_hash != CHECKSUMS[BINARY_URL]:
|
| 109 |
+
raise ChecksumMismatchError()
|
| 110 |
+
|
| 111 |
+
def start_tunnel(self) -> str:
|
| 112 |
+
self.download_binary()
|
| 113 |
+
self.url = self._start_tunnel(BINARY_PATH)
|
| 114 |
+
return self.url
|
| 115 |
+
|
| 116 |
+
def kill(self):
|
| 117 |
+
if self.proc is not None:
|
| 118 |
+
print(f"Killing tunnel {self.local_host}:{self.local_port} <> {self.url}")
|
| 119 |
+
self.proc.terminate()
|
| 120 |
+
self.proc = None
|
| 121 |
+
|
| 122 |
+
def _start_tunnel(self, binary: str) -> str:
|
| 123 |
+
CURRENT_TUNNELS.append(self)
|
| 124 |
+
command = [
|
| 125 |
+
binary,
|
| 126 |
+
"http",
|
| 127 |
+
"-n",
|
| 128 |
+
self.share_token,
|
| 129 |
+
"-l",
|
| 130 |
+
str(self.local_port),
|
| 131 |
+
"-i",
|
| 132 |
+
self.local_host,
|
| 133 |
+
"--uc",
|
| 134 |
+
"--sd",
|
| 135 |
+
"random",
|
| 136 |
+
"--ue",
|
| 137 |
+
"--server_addr",
|
| 138 |
+
f"{self.remote_host}:{self.remote_port}",
|
| 139 |
+
"--disable_log_color",
|
| 140 |
+
]
|
| 141 |
+
if self.share_server_tls_certificate is not None:
|
| 142 |
+
command.extend(
|
| 143 |
+
[
|
| 144 |
+
"--tls_enable",
|
| 145 |
+
"--tls_trusted_ca_file",
|
| 146 |
+
self.share_server_tls_certificate,
|
| 147 |
+
]
|
| 148 |
+
)
|
| 149 |
+
self.proc = subprocess.Popen(
|
| 150 |
+
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
| 151 |
+
)
|
| 152 |
+
atexit.register(self.kill)
|
| 153 |
+
return self._read_url_from_tunnel_stream()
|
| 154 |
+
|
| 155 |
+
def _read_url_from_tunnel_stream(self) -> str:
|
| 156 |
+
start_timestamp = time.time()
|
| 157 |
+
|
| 158 |
+
log = []
|
| 159 |
+
url = ""
|
| 160 |
+
|
| 161 |
+
def _raise_tunnel_error():
|
| 162 |
+
log_text = "\n".join(log)
|
| 163 |
+
print(log_text, file=sys.stderr)
|
| 164 |
+
raise ValueError(f"{TUNNEL_ERROR_MESSAGE}\n{log_text}")
|
| 165 |
+
|
| 166 |
+
while url == "":
|
| 167 |
+
if time.time() - start_timestamp >= TUNNEL_TIMEOUT_SECONDS:
|
| 168 |
+
_raise_tunnel_error()
|
| 169 |
+
|
| 170 |
+
assert self.proc is not None # noqa: S101
|
| 171 |
+
if self.proc.stdout is None:
|
| 172 |
+
continue
|
| 173 |
+
|
| 174 |
+
line = self.proc.stdout.readline()
|
| 175 |
+
line = line.decode("utf-8")
|
| 176 |
+
|
| 177 |
+
if line == "":
|
| 178 |
+
continue
|
| 179 |
+
|
| 180 |
+
log.append(line.strip())
|
| 181 |
+
|
| 182 |
+
if "start proxy success" in line:
|
| 183 |
+
result = re.search("start proxy success: (.+)\n", line)
|
| 184 |
+
if result is None:
|
| 185 |
+
_raise_tunnel_error()
|
| 186 |
+
else:
|
| 187 |
+
url = result.group(1)
|
| 188 |
+
elif "login to server failed" in line:
|
| 189 |
+
_raise_tunnel_error()
|
| 190 |
+
|
| 191 |
+
return url
|
trackio/agent_sessions.py
ADDED
|
@@ -0,0 +1,533 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Agent session serialization for the Hugging Face Hub's trace viewer.
|
| 2 |
+
|
| 3 |
+
The Hub renders `.jsonl` agent sessions in a dedicated trace viewer, for both
|
| 4 |
+
Datasets and Storage Buckets. This module is the single place Trackio produces
|
| 5 |
+
that format: the logbook's attached agent sessions and the `traces` table both
|
| 6 |
+
serialize through it.
|
| 7 |
+
|
| 8 |
+
Trackio emits **Pi's session format** (version 3), which the Hub supports
|
| 9 |
+
natively. The Hub also documents a Session Trace Simple Format (STS), but as of
|
| 10 |
+
this writing files written to that spec are not rendered by the viewer — a
|
| 11 |
+
controlled test (same bucket, same folder, one variable) showed a Codex-native
|
| 12 |
+
and a Pi-format session rendering while a spec-compliant STS session displayed
|
| 13 |
+
as raw JSONL, and none of the public datasets tagged `format:agent-traces` use
|
| 14 |
+
STS. Pi's format is also the better fit: `id`/`parentId`, token `usage`, `cost`,
|
| 15 |
+
and `isError` all have native homes, where STS had none.
|
| 16 |
+
|
| 17 |
+
A session file is a header line followed by one entry per line:
|
| 18 |
+
|
| 19 |
+
{"type":"session","version":3,"id":"...","timestamp":"...","cwd":"..."}
|
| 20 |
+
{"type":"message","id":"...","parentId":null,"timestamp":"...",
|
| 21 |
+
"message":{"role":"user","content":[{"type":"text","text":"..."}]}}
|
| 22 |
+
|
| 23 |
+
Entries chain through `parentId`. Message content is a list of typed blocks
|
| 24 |
+
(`text`, `thinking`, `toolCall`); tool results are their own message with
|
| 25 |
+
`role: "toolResult"`, a `toolCallId`, a `toolName`, and an optional `isError`.
|
| 26 |
+
|
| 27 |
+
Format reference:
|
| 28 |
+
https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/session-format.md
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
import hashlib
|
| 34 |
+
import json
|
| 35 |
+
import re
|
| 36 |
+
import uuid
|
| 37 |
+
from datetime import datetime, timezone
|
| 38 |
+
from pathlib import Path
|
| 39 |
+
from typing import Any
|
| 40 |
+
|
| 41 |
+
HARNESS = "trackio"
|
| 42 |
+
SESSION_VERSION = 3
|
| 43 |
+
SESSION_NAME_MAX_CHARS = 120
|
| 44 |
+
|
| 45 |
+
# Formats the Hub renders without conversion.
|
| 46 |
+
HUB_NATIVE_PROVIDERS = {"Codex", "Claude Code", "Pi"}
|
| 47 |
+
|
| 48 |
+
# Pi roles. Trackio's OpenAI-style `system` maps onto `developer`, which is what
|
| 49 |
+
# the sessions that render in the viewer use.
|
| 50 |
+
ROLE_BY_OPENAI_ROLE = {
|
| 51 |
+
"system": "developer",
|
| 52 |
+
"developer": "developer",
|
| 53 |
+
"user": "user",
|
| 54 |
+
"assistant": "assistant",
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
_NUMERIC_RE = re.compile(r"\d+(?:\.\d+)?")
|
| 58 |
+
_UNSAFE_ID_RE = re.compile(r"[^a-zA-Z0-9._-]+")
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def parse_time(value: str | int | float | None) -> datetime | None:
|
| 62 |
+
"""Parse an ISO-8601 string or an epoch number (seconds or milliseconds)."""
|
| 63 |
+
if value in (None, ""):
|
| 64 |
+
return None
|
| 65 |
+
if isinstance(value, (int, float)) or (
|
| 66 |
+
isinstance(value, str) and _NUMERIC_RE.fullmatch(value)
|
| 67 |
+
):
|
| 68 |
+
numeric = float(value)
|
| 69 |
+
if numeric > 10_000_000_000:
|
| 70 |
+
numeric /= 1000
|
| 71 |
+
try:
|
| 72 |
+
return datetime.fromtimestamp(numeric, tz=timezone.utc)
|
| 73 |
+
except (OverflowError, OSError, ValueError):
|
| 74 |
+
return None
|
| 75 |
+
try:
|
| 76 |
+
return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
| 77 |
+
except ValueError:
|
| 78 |
+
return None
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def iso_z(value: str | int | float | None) -> str | None:
|
| 82 |
+
"""An ISO-8601 UTC timestamp with the `Z` suffix Pi sessions use."""
|
| 83 |
+
parsed = parse_time(value)
|
| 84 |
+
if parsed is None:
|
| 85 |
+
return None
|
| 86 |
+
if parsed.tzinfo is None:
|
| 87 |
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
| 88 |
+
return (
|
| 89 |
+
parsed.astimezone(timezone.utc)
|
| 90 |
+
.isoformat(timespec="milliseconds")
|
| 91 |
+
.replace("+00:00", "Z")
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def entry_id(*parts: Any) -> str:
|
| 96 |
+
"""A stable 8-character entry id, matching the width Pi sessions use."""
|
| 97 |
+
seed = ":".join(str(part) for part in parts)
|
| 98 |
+
return hashlib.sha1(seed.encode("utf-8")).hexdigest()[:8]
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def safe_id(value: str) -> str:
|
| 102 |
+
"""A filesystem- and bucket-safe form of an identifier."""
|
| 103 |
+
cleaned = _UNSAFE_ID_RE.sub("-", str(value)).strip("-.")
|
| 104 |
+
return cleaned[:100] or f"session-{uuid.uuid4().hex[:12]}"
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def is_hub_native_jsonl(path: Path, provider: str | None = None) -> bool:
|
| 108 |
+
"""Whether the Hub's trace viewer renders this file without conversion.
|
| 109 |
+
|
| 110 |
+
Only the harness-native formats qualify. A session header carrying a
|
| 111 |
+
`harness` but no `version` is STS, which the viewer does not render, so it
|
| 112 |
+
still needs converting.
|
| 113 |
+
"""
|
| 114 |
+
if path.suffix.lower() != ".jsonl":
|
| 115 |
+
return False
|
| 116 |
+
if provider in HUB_NATIVE_PROVIDERS:
|
| 117 |
+
return True
|
| 118 |
+
try:
|
| 119 |
+
with path.open(encoding="utf-8") as handle:
|
| 120 |
+
first_line = next((line for line in handle if line.strip()), "")
|
| 121 |
+
header = json.loads(first_line)
|
| 122 |
+
except (OSError, json.JSONDecodeError):
|
| 123 |
+
return False
|
| 124 |
+
if not isinstance(header, dict):
|
| 125 |
+
return False
|
| 126 |
+
if header.get("type") == "session" and header.get("version") is not None:
|
| 127 |
+
return True
|
| 128 |
+
# Codex sessions open with a `session_meta` envelope rather than `session`.
|
| 129 |
+
return header.get("type") == "session_meta" and isinstance(
|
| 130 |
+
header.get("payload"), dict
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def session_header(
|
| 135 |
+
session_id: str,
|
| 136 |
+
timestamp: str | None = None,
|
| 137 |
+
name: str | None = None,
|
| 138 |
+
cwd: str | None = None,
|
| 139 |
+
**extra: Any,
|
| 140 |
+
) -> dict[str, Any]:
|
| 141 |
+
header: dict[str, Any] = {
|
| 142 |
+
"type": "session",
|
| 143 |
+
"version": SESSION_VERSION,
|
| 144 |
+
"id": str(session_id),
|
| 145 |
+
"timestamp": iso_z(timestamp) or iso_z(datetime.now(timezone.utc)),
|
| 146 |
+
"cwd": cwd or "/",
|
| 147 |
+
"harness": HARNESS,
|
| 148 |
+
}
|
| 149 |
+
if name:
|
| 150 |
+
header["name"] = name
|
| 151 |
+
header.update({key: value for key, value in extra.items() if value is not None})
|
| 152 |
+
return header
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def dumps(records: list[dict[str, Any]]) -> str:
|
| 156 |
+
return (
|
| 157 |
+
"\n".join(json.dumps(record, ensure_ascii=False) for record in records) + "\n"
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def write_session(path: Path, records: list[dict[str, Any]]) -> None:
|
| 162 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 163 |
+
path.write_text(dumps(records), encoding="utf-8")
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def content_text(value: Any) -> str:
|
| 167 |
+
"""Flatten an OpenAI-style `content` value to text.
|
| 168 |
+
|
| 169 |
+
Content may be a plain string, a list of typed blocks, or a serialized
|
| 170 |
+
Trackio media object. Non-text blocks keep a readable placeholder rather
|
| 171 |
+
than being dropped, so the viewer shows that something was there.
|
| 172 |
+
"""
|
| 173 |
+
if value is None:
|
| 174 |
+
return ""
|
| 175 |
+
if isinstance(value, str):
|
| 176 |
+
return value
|
| 177 |
+
if isinstance(value, list):
|
| 178 |
+
parts = [content_text(item) for item in value]
|
| 179 |
+
return "\n".join(part for part in parts if part)
|
| 180 |
+
if isinstance(value, dict):
|
| 181 |
+
media_type = value.get("_type")
|
| 182 |
+
if isinstance(media_type, str) and media_type.startswith("trackio."):
|
| 183 |
+
label = media_type.split(".", 1)[1]
|
| 184 |
+
caption = value.get("caption") or value.get("file_path") or ""
|
| 185 |
+
return f"[{label}{': ' + str(caption) if caption else ''}]"
|
| 186 |
+
for key in ("text", "content", "output", "value"):
|
| 187 |
+
if isinstance(value.get(key), str):
|
| 188 |
+
return value[key]
|
| 189 |
+
return json.dumps(value, ensure_ascii=False)
|
| 190 |
+
return str(value)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def text_block(text: str) -> dict[str, Any]:
|
| 194 |
+
return {"type": "text", "text": text}
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def thinking_block(text: str) -> dict[str, Any]:
|
| 198 |
+
return {"type": "thinking", "thinking": text}
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def tool_call_block(call_id: str, name: str, arguments: Any) -> dict[str, Any]:
|
| 202 |
+
"""A `toolCall` block. Unlike STS, `arguments` stays a real object."""
|
| 203 |
+
if isinstance(arguments, str):
|
| 204 |
+
try:
|
| 205 |
+
arguments = json.loads(arguments)
|
| 206 |
+
except json.JSONDecodeError:
|
| 207 |
+
arguments = {"input": arguments}
|
| 208 |
+
if not isinstance(arguments, dict):
|
| 209 |
+
arguments = {} if arguments is None else {"input": arguments}
|
| 210 |
+
return {
|
| 211 |
+
"type": "toolCall",
|
| 212 |
+
"id": str(call_id),
|
| 213 |
+
"name": str(name or "tool"),
|
| 214 |
+
"arguments": arguments,
|
| 215 |
+
}
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
class _Chain:
|
| 219 |
+
"""Accumulates entries, chaining each to the one before via `parentId`."""
|
| 220 |
+
|
| 221 |
+
def __init__(self, session_id: str, default_timestamp: str | None):
|
| 222 |
+
self.session_id = session_id
|
| 223 |
+
self.default_timestamp = default_timestamp
|
| 224 |
+
self.records: list[dict[str, Any]] = []
|
| 225 |
+
self._parent: str | None = None
|
| 226 |
+
|
| 227 |
+
def add(self, entry: dict[str, Any], timestamp: str | None = None) -> str:
|
| 228 |
+
entry["parentId"] = self._parent
|
| 229 |
+
entry["timestamp"] = iso_z(timestamp) or self.default_timestamp
|
| 230 |
+
self._parent = entry["id"]
|
| 231 |
+
self.records.append(entry)
|
| 232 |
+
return entry["id"]
|
| 233 |
+
|
| 234 |
+
def message(
|
| 235 |
+
self, index: Any, message: dict[str, Any], timestamp: str | None = None
|
| 236 |
+
) -> str:
|
| 237 |
+
return self.add(
|
| 238 |
+
{
|
| 239 |
+
"type": "message",
|
| 240 |
+
"id": entry_id(self.session_id, "msg", index),
|
| 241 |
+
"message": message,
|
| 242 |
+
},
|
| 243 |
+
timestamp,
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def span_usage(span: dict[str, Any]) -> dict[str, Any] | None:
|
| 248 |
+
"""Pi's `usage` block from a Trackio span's `usage` and `cost_usd`."""
|
| 249 |
+
usage = span.get("usage") or {}
|
| 250 |
+
raw_input = usage.get("input_tokens", usage.get("prompt_tokens"))
|
| 251 |
+
raw_output = usage.get("output_tokens", usage.get("completion_tokens"))
|
| 252 |
+
cost = span.get("cost_usd")
|
| 253 |
+
if raw_input is None and raw_output is None and cost is None:
|
| 254 |
+
return None
|
| 255 |
+
try:
|
| 256 |
+
input_tokens = int(raw_input or 0)
|
| 257 |
+
output_tokens = int(raw_output or 0)
|
| 258 |
+
except (TypeError, ValueError):
|
| 259 |
+
input_tokens = output_tokens = 0
|
| 260 |
+
total = usage.get("total_tokens") or (input_tokens + output_tokens)
|
| 261 |
+
block: dict[str, Any] = {
|
| 262 |
+
"input": input_tokens,
|
| 263 |
+
"output": output_tokens,
|
| 264 |
+
"cacheRead": int(usage.get("cache_read_input_tokens") or 0),
|
| 265 |
+
"cacheWrite": int(usage.get("cache_creation_input_tokens") or 0),
|
| 266 |
+
"totalTokens": total,
|
| 267 |
+
}
|
| 268 |
+
if cost is not None:
|
| 269 |
+
try:
|
| 270 |
+
block["cost"] = {
|
| 271 |
+
"input": 0,
|
| 272 |
+
"output": 0,
|
| 273 |
+
"cacheRead": 0,
|
| 274 |
+
"cacheWrite": 0,
|
| 275 |
+
"total": float(cost),
|
| 276 |
+
}
|
| 277 |
+
except (TypeError, ValueError):
|
| 278 |
+
pass
|
| 279 |
+
return block
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def _span_is_error(span: dict[str, Any]) -> bool:
|
| 283 |
+
status = str(span.get("status") or "").lower()
|
| 284 |
+
return status in {"error", "failed"} or bool(span.get("error"))
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def _message_is_error(message: dict[str, Any]) -> bool:
|
| 288 |
+
if message.get("is_error") is True or message.get("isError") is True:
|
| 289 |
+
return True
|
| 290 |
+
if message.get("error"):
|
| 291 |
+
return True
|
| 292 |
+
return str(message.get("status") or "").lower() in {"error", "failed"}
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def _generation_output_text(span: dict[str, Any]) -> str:
|
| 296 |
+
output = span.get("output")
|
| 297 |
+
if isinstance(output, dict):
|
| 298 |
+
for key in ("text", "content", "message", "output"):
|
| 299 |
+
if output.get(key) is not None:
|
| 300 |
+
return content_text(output[key])
|
| 301 |
+
return content_text(output)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def _span_sort_key(indexed_span: tuple[int, dict[str, Any]]) -> tuple[float, int]:
|
| 305 |
+
index, span = indexed_span
|
| 306 |
+
started = parse_time(span.get("start_time") or span.get("startTime"))
|
| 307 |
+
return (started.timestamp() if started else float("inf"), index)
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def _ordered_spans(spans: list[dict[str, Any]], kind: str) -> list[dict[str, Any]]:
|
| 311 |
+
indexed = [
|
| 312 |
+
(index, span)
|
| 313 |
+
for index, span in enumerate(spans)
|
| 314 |
+
if str(span.get("kind") or "").lower() == kind
|
| 315 |
+
]
|
| 316 |
+
return [span for _, span in sorted(indexed, key=_span_sort_key)]
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def _session_name(records: list[dict[str, Any]], fallback: str) -> str:
|
| 320 |
+
"""Title for the session: the request, when there is one."""
|
| 321 |
+
for wanted_role in ("user", None):
|
| 322 |
+
for record in records:
|
| 323 |
+
message = record.get("message") or {}
|
| 324 |
+
role = str(message.get("role") or "")
|
| 325 |
+
if wanted_role is not None and role != wanted_role:
|
| 326 |
+
continue
|
| 327 |
+
text = " ".join(content_text(message.get("content")).split())
|
| 328 |
+
if text:
|
| 329 |
+
return text[:SESSION_NAME_MAX_CHARS]
|
| 330 |
+
return fallback
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def _messages_to_entries(
|
| 334 |
+
chain: _Chain,
|
| 335 |
+
messages: list[dict[str, Any]],
|
| 336 |
+
generations: list[dict[str, Any]],
|
| 337 |
+
tool_spans: list[dict[str, Any]],
|
| 338 |
+
) -> None:
|
| 339 |
+
"""Emit message entries, enriched with what the spans know.
|
| 340 |
+
|
| 341 |
+
Spans supply what the message layer cannot: token usage and cost on the
|
| 342 |
+
generation that produced each assistant turn, and failure status on a tool
|
| 343 |
+
result. Tool call ids and span ids come from different namespaces, so a
|
| 344 |
+
result is matched to its span by call id first and by order second.
|
| 345 |
+
"""
|
| 346 |
+
tool_names: dict[str, str] = {}
|
| 347 |
+
tool_span_by_id = {str(span.get("id")): span for span in tool_spans}
|
| 348 |
+
remaining_tool_spans = list(tool_spans)
|
| 349 |
+
generation_index = 0
|
| 350 |
+
|
| 351 |
+
for index, message in enumerate(messages):
|
| 352 |
+
role = str(message.get("role") or "assistant").lower()
|
| 353 |
+
timestamp = message.get("timestamp") or message.get("created_at")
|
| 354 |
+
|
| 355 |
+
if role == "tool":
|
| 356 |
+
call_id = str(
|
| 357 |
+
message.get("tool_call_id")
|
| 358 |
+
or message.get("toolCallId")
|
| 359 |
+
or f"call-{index}"
|
| 360 |
+
)
|
| 361 |
+
span = tool_span_by_id.get(call_id)
|
| 362 |
+
if span is None and remaining_tool_spans:
|
| 363 |
+
span = remaining_tool_spans[0]
|
| 364 |
+
if span is not None and span in remaining_tool_spans:
|
| 365 |
+
remaining_tool_spans.remove(span)
|
| 366 |
+
result: dict[str, Any] = {
|
| 367 |
+
"role": "toolResult",
|
| 368 |
+
"toolCallId": call_id,
|
| 369 |
+
"toolName": tool_names.get(call_id)
|
| 370 |
+
or str((span or {}).get("name") or "tool"),
|
| 371 |
+
"content": [text_block(content_text(message.get("content")))],
|
| 372 |
+
}
|
| 373 |
+
if _message_is_error(message) or (
|
| 374 |
+
span is not None and _span_is_error(span)
|
| 375 |
+
):
|
| 376 |
+
result["isError"] = True
|
| 377 |
+
chain.message(index, result, timestamp)
|
| 378 |
+
continue
|
| 379 |
+
|
| 380 |
+
blocks: list[dict[str, Any]] = []
|
| 381 |
+
reasoning = (
|
| 382 |
+
message.get("reasoning_content")
|
| 383 |
+
or message.get("reasoningContent")
|
| 384 |
+
or message.get("reasoning")
|
| 385 |
+
)
|
| 386 |
+
if reasoning:
|
| 387 |
+
blocks.append(thinking_block(content_text(reasoning)))
|
| 388 |
+
text = content_text(message.get("content"))
|
| 389 |
+
if text:
|
| 390 |
+
blocks.append(text_block(text))
|
| 391 |
+
|
| 392 |
+
calls = list(message.get("tool_calls") or message.get("toolCalls") or [])
|
| 393 |
+
if message.get("function_call"):
|
| 394 |
+
calls.append({"function": message["function_call"]})
|
| 395 |
+
for call_index, call in enumerate(calls):
|
| 396 |
+
if not isinstance(call, dict):
|
| 397 |
+
continue
|
| 398 |
+
function = (
|
| 399 |
+
call.get("function") if isinstance(call.get("function"), dict) else call
|
| 400 |
+
)
|
| 401 |
+
call_id = str(call.get("id") or f"call-{index}-{call_index}")
|
| 402 |
+
name = str(function.get("name") or "tool")
|
| 403 |
+
tool_names[call_id] = name
|
| 404 |
+
blocks.append(tool_call_block(call_id, name, function.get("arguments")))
|
| 405 |
+
|
| 406 |
+
entry_message: dict[str, Any] = {
|
| 407 |
+
"role": ROLE_BY_OPENAI_ROLE.get(role, "assistant"),
|
| 408 |
+
"content": blocks,
|
| 409 |
+
}
|
| 410 |
+
if entry_message["role"] == "assistant":
|
| 411 |
+
# Pair the Nth assistant turn with the Nth generation span rather
|
| 412 |
+
# than repeating one span's totals on every turn, which would make
|
| 413 |
+
# the viewer double-count tokens and cost.
|
| 414 |
+
if generation_index < len(generations):
|
| 415 |
+
usage = span_usage(generations[generation_index])
|
| 416 |
+
if usage:
|
| 417 |
+
entry_message["usage"] = usage
|
| 418 |
+
generation_index += 1
|
| 419 |
+
chain.message(index, entry_message, timestamp)
|
| 420 |
+
|
| 421 |
+
|
| 422 |
+
def _spans_to_entries(
|
| 423 |
+
chain: _Chain, spans: list[dict[str, Any]], generations: list[dict[str, Any]]
|
| 424 |
+
) -> None:
|
| 425 |
+
"""Emit entries for a trace that carries spans but no messages."""
|
| 426 |
+
ordered = sorted(
|
| 427 |
+
((index, span) for index, span in enumerate(spans) if isinstance(span, dict)),
|
| 428 |
+
key=_span_sort_key,
|
| 429 |
+
)
|
| 430 |
+
for index, span in ordered:
|
| 431 |
+
kind = str(span.get("kind") or "span").lower()
|
| 432 |
+
timestamp = span.get("start_time") or span.get("startTime")
|
| 433 |
+
if kind == "generation":
|
| 434 |
+
message: dict[str, Any] = {
|
| 435 |
+
"role": "assistant",
|
| 436 |
+
"content": [text_block(_generation_output_text(span))],
|
| 437 |
+
}
|
| 438 |
+
usage = span_usage(span)
|
| 439 |
+
if usage:
|
| 440 |
+
message["usage"] = usage
|
| 441 |
+
chain.message(f"span-{index}", message, timestamp)
|
| 442 |
+
elif kind == "tool":
|
| 443 |
+
call_id = str(span.get("id") or f"span-tool-{index}")
|
| 444 |
+
name = str(span.get("name") or "tool")
|
| 445 |
+
chain.message(
|
| 446 |
+
f"span-{index}-call",
|
| 447 |
+
{
|
| 448 |
+
"role": "assistant",
|
| 449 |
+
"content": [tool_call_block(call_id, name, span.get("input"))],
|
| 450 |
+
},
|
| 451 |
+
timestamp,
|
| 452 |
+
)
|
| 453 |
+
result: dict[str, Any] = {
|
| 454 |
+
"role": "toolResult",
|
| 455 |
+
"toolCallId": call_id,
|
| 456 |
+
"toolName": name,
|
| 457 |
+
"content": [
|
| 458 |
+
text_block(content_text(span.get("error") or span.get("output")))
|
| 459 |
+
],
|
| 460 |
+
}
|
| 461 |
+
if _span_is_error(span):
|
| 462 |
+
result["isError"] = True
|
| 463 |
+
chain.message(f"span-{index}-result", result, timestamp)
|
| 464 |
+
else:
|
| 465 |
+
chain.message(
|
| 466 |
+
f"span-{index}",
|
| 467 |
+
{
|
| 468 |
+
"role": "developer",
|
| 469 |
+
"content": [text_block(str(span.get("name") or kind))],
|
| 470 |
+
},
|
| 471 |
+
timestamp,
|
| 472 |
+
)
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def trace_to_records(
|
| 476 |
+
trace: dict[str, Any], project: str | None = None
|
| 477 |
+
) -> list[dict[str, Any]]:
|
| 478 |
+
"""Serialize one Trackio trace row into a Pi-format agent session.
|
| 479 |
+
|
| 480 |
+
The entries form the linear conversation the viewer renders, enriched from
|
| 481 |
+
the spans with model, token usage, cost, and failure status. The full span
|
| 482 |
+
tree — hierarchy, `duration_ms`, per-span `metadata`, and the `kind` of
|
| 483 |
+
non-tool spans — stays in SQLite, which remains authoritative for every
|
| 484 |
+
Trackio reader.
|
| 485 |
+
"""
|
| 486 |
+
messages = [m for m in (trace.get("messages") or []) if isinstance(m, dict)]
|
| 487 |
+
spans = [s for s in (trace.get("spans") or []) if isinstance(s, dict)]
|
| 488 |
+
generations = _ordered_spans(spans, "generation")
|
| 489 |
+
tool_spans = _ordered_spans(spans, "tool")
|
| 490 |
+
|
| 491 |
+
trace_id = str(trace.get("id") or uuid.uuid4().hex)
|
| 492 |
+
timestamp = iso_z(trace.get("timestamp"))
|
| 493 |
+
chain = _Chain(trace_id, timestamp)
|
| 494 |
+
|
| 495 |
+
model = next(
|
| 496 |
+
(str(span["model"]) for span in generations if span.get("model")),
|
| 497 |
+
None,
|
| 498 |
+
)
|
| 499 |
+
if model:
|
| 500 |
+
chain.add(
|
| 501 |
+
{
|
| 502 |
+
"type": "model_change",
|
| 503 |
+
"id": entry_id(trace_id, "model"),
|
| 504 |
+
"modelId": model,
|
| 505 |
+
},
|
| 506 |
+
trace.get("timestamp"),
|
| 507 |
+
)
|
| 508 |
+
|
| 509 |
+
if messages:
|
| 510 |
+
_messages_to_entries(chain, messages, generations, tool_spans)
|
| 511 |
+
else:
|
| 512 |
+
_spans_to_entries(chain, spans, generations)
|
| 513 |
+
|
| 514 |
+
header = session_header(
|
| 515 |
+
trace_id,
|
| 516 |
+
timestamp=trace.get("timestamp"),
|
| 517 |
+
name=_session_name(chain.records, trace_id),
|
| 518 |
+
cwd=f"/{project}" if project else "/",
|
| 519 |
+
trackio={
|
| 520 |
+
"project": project,
|
| 521 |
+
"run_id": trace.get("run_id"),
|
| 522 |
+
"run_name": trace.get("run_name"),
|
| 523 |
+
"step": trace.get("step"),
|
| 524 |
+
"key": trace.get("key"),
|
| 525 |
+
"trace_index": trace.get("trace_index"),
|
| 526 |
+
"metadata": trace.get("metadata") or {},
|
| 527 |
+
},
|
| 528 |
+
)
|
| 529 |
+
return [header, *chain.records]
|
| 530 |
+
|
| 531 |
+
|
| 532 |
+
def trace_session_filename(trace_id: Any) -> str:
|
| 533 |
+
return f"{safe_id(str(trace_id))}.jsonl"
|
trackio/alerts.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import logging
|
| 3 |
+
import ssl
|
| 4 |
+
import urllib.request
|
| 5 |
+
from enum import Enum
|
| 6 |
+
|
| 7 |
+
try:
|
| 8 |
+
import certifi
|
| 9 |
+
|
| 10 |
+
_SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where())
|
| 11 |
+
except ImportError:
|
| 12 |
+
_SSL_CONTEXT = None
|
| 13 |
+
|
| 14 |
+
logger = logging.getLogger(__name__)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class AlertLevel(str, Enum):
|
| 18 |
+
INFO = "info"
|
| 19 |
+
WARN = "warn"
|
| 20 |
+
ERROR = "error"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
ALERT_LEVEL_ORDER = {
|
| 24 |
+
AlertLevel.INFO: 0,
|
| 25 |
+
AlertLevel.WARN: 1,
|
| 26 |
+
AlertLevel.ERROR: 2,
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
ALERT_COLORS = {
|
| 30 |
+
AlertLevel.INFO: "\033[94m",
|
| 31 |
+
AlertLevel.WARN: "\033[93m",
|
| 32 |
+
AlertLevel.ERROR: "\033[91m",
|
| 33 |
+
}
|
| 34 |
+
RESET_COLOR = "\033[0m"
|
| 35 |
+
|
| 36 |
+
LEVEL_EMOJI = {
|
| 37 |
+
AlertLevel.INFO: "ℹ️",
|
| 38 |
+
AlertLevel.WARN: "⚠️",
|
| 39 |
+
AlertLevel.ERROR: "🚨",
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def format_alert_terminal(
|
| 44 |
+
level: AlertLevel, title: str, text: str | None, step: int | None
|
| 45 |
+
) -> str:
|
| 46 |
+
color = ALERT_COLORS.get(level, "")
|
| 47 |
+
step_str = f" (step {step})" if step is not None else ""
|
| 48 |
+
if text:
|
| 49 |
+
return f"{color}[TRACKIO {level.value.upper()}]{RESET_COLOR} {title}: {text}{step_str}"
|
| 50 |
+
return f"{color}[TRACKIO {level.value.upper()}]{RESET_COLOR} {title}{step_str}"
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _is_slack_url(url: str) -> bool:
|
| 54 |
+
return "hooks.slack.com" in url
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _is_discord_url(url: str) -> bool:
|
| 58 |
+
return "discord.com/api/webhooks" in url or "discordapp.com/api/webhooks" in url
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _build_slack_payload(
|
| 62 |
+
level: AlertLevel,
|
| 63 |
+
title: str,
|
| 64 |
+
text: str | None,
|
| 65 |
+
project: str,
|
| 66 |
+
run: str,
|
| 67 |
+
step: int | None,
|
| 68 |
+
) -> dict:
|
| 69 |
+
emoji = LEVEL_EMOJI.get(level, "")
|
| 70 |
+
step_str = f" • Step {step}" if step is not None else ""
|
| 71 |
+
header = f"{emoji} *[{level.value.upper()}] {title}*"
|
| 72 |
+
context = f"Project: {project} • Run: {run}{step_str}"
|
| 73 |
+
blocks = [
|
| 74 |
+
{"type": "section", "text": {"type": "mrkdwn", "text": header}},
|
| 75 |
+
]
|
| 76 |
+
if text:
|
| 77 |
+
blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": text}})
|
| 78 |
+
blocks.append(
|
| 79 |
+
{"type": "context", "elements": [{"type": "mrkdwn", "text": context}]}
|
| 80 |
+
)
|
| 81 |
+
return {"blocks": blocks}
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def _build_discord_payload(
|
| 85 |
+
level: AlertLevel,
|
| 86 |
+
title: str,
|
| 87 |
+
text: str | None,
|
| 88 |
+
project: str,
|
| 89 |
+
run: str,
|
| 90 |
+
step: int | None,
|
| 91 |
+
) -> dict:
|
| 92 |
+
color_map = {
|
| 93 |
+
AlertLevel.INFO: 3447003,
|
| 94 |
+
AlertLevel.WARN: 16776960,
|
| 95 |
+
AlertLevel.ERROR: 15158332,
|
| 96 |
+
}
|
| 97 |
+
emoji = LEVEL_EMOJI.get(level, "")
|
| 98 |
+
step_str = f" • Step {step}" if step is not None else ""
|
| 99 |
+
embed = {
|
| 100 |
+
"title": f"{emoji} [{level.value.upper()}] {title}",
|
| 101 |
+
"color": color_map.get(level, 0),
|
| 102 |
+
"footer": {"text": f"Project: {project} • Run: {run}{step_str}"},
|
| 103 |
+
}
|
| 104 |
+
if text:
|
| 105 |
+
embed["description"] = text
|
| 106 |
+
return {"embeds": [embed]}
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _build_generic_payload(
|
| 110 |
+
level: AlertLevel,
|
| 111 |
+
title: str,
|
| 112 |
+
text: str | None,
|
| 113 |
+
project: str,
|
| 114 |
+
run: str,
|
| 115 |
+
step: int | None,
|
| 116 |
+
timestamp: str | None,
|
| 117 |
+
) -> dict:
|
| 118 |
+
return {
|
| 119 |
+
"level": level.value,
|
| 120 |
+
"title": title,
|
| 121 |
+
"text": text,
|
| 122 |
+
"project": project,
|
| 123 |
+
"run": run,
|
| 124 |
+
"step": step,
|
| 125 |
+
"timestamp": timestamp,
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def parse_alert_level(level: AlertLevel | str) -> AlertLevel:
|
| 130 |
+
if isinstance(level, AlertLevel):
|
| 131 |
+
return level
|
| 132 |
+
normalized = level.lower().strip()
|
| 133 |
+
try:
|
| 134 |
+
return AlertLevel(normalized)
|
| 135 |
+
except ValueError as e:
|
| 136 |
+
allowed = ", ".join(lvl.value for lvl in AlertLevel)
|
| 137 |
+
raise ValueError(
|
| 138 |
+
f"Invalid alert level '{level}'. Expected one of: {allowed}."
|
| 139 |
+
) from e
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def resolve_webhook_min_level(
|
| 143 |
+
webhook_min_level: AlertLevel | str | None,
|
| 144 |
+
) -> AlertLevel | None:
|
| 145 |
+
if webhook_min_level is None:
|
| 146 |
+
return None
|
| 147 |
+
return parse_alert_level(webhook_min_level)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def should_send_webhook(
|
| 151 |
+
level: AlertLevel, webhook_min_level: AlertLevel | None
|
| 152 |
+
) -> bool:
|
| 153 |
+
if webhook_min_level is None:
|
| 154 |
+
return True
|
| 155 |
+
return ALERT_LEVEL_ORDER[level] >= ALERT_LEVEL_ORDER[webhook_min_level]
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def send_webhook(
|
| 159 |
+
url: str,
|
| 160 |
+
level: AlertLevel,
|
| 161 |
+
title: str,
|
| 162 |
+
text: str | None,
|
| 163 |
+
project: str,
|
| 164 |
+
run: str,
|
| 165 |
+
step: int | None,
|
| 166 |
+
timestamp: str | None = None,
|
| 167 |
+
) -> None:
|
| 168 |
+
if _is_slack_url(url):
|
| 169 |
+
payload = _build_slack_payload(level, title, text, project, run, step)
|
| 170 |
+
elif _is_discord_url(url):
|
| 171 |
+
payload = _build_discord_payload(level, title, text, project, run, step)
|
| 172 |
+
else:
|
| 173 |
+
payload = _build_generic_payload(
|
| 174 |
+
level, title, text, project, run, step, timestamp
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
data = json.dumps(payload).encode("utf-8")
|
| 178 |
+
req = urllib.request.Request(
|
| 179 |
+
url, data=data, headers={"Content-Type": "application/json"}
|
| 180 |
+
)
|
| 181 |
+
try:
|
| 182 |
+
urllib.request.urlopen(req, timeout=10, context=_SSL_CONTEXT)
|
| 183 |
+
except Exception as e:
|
| 184 |
+
logger.warning(f"Failed to send webhook to {url}: {e}")
|
trackio/api.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Iterator
|
| 2 |
+
|
| 3 |
+
from trackio.registry import Registry
|
| 4 |
+
from trackio.sqlite_storage import SQLiteStorage
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class Run:
|
| 8 |
+
def __init__(self, project: str, name: str, run_id: str | None = None):
|
| 9 |
+
self.project = project
|
| 10 |
+
self.name = name
|
| 11 |
+
self._id = run_id or name
|
| 12 |
+
self._config = None
|
| 13 |
+
|
| 14 |
+
@property
|
| 15 |
+
def id(self) -> str:
|
| 16 |
+
return self._id
|
| 17 |
+
|
| 18 |
+
@property
|
| 19 |
+
def config(self) -> dict | None:
|
| 20 |
+
if self._config is None:
|
| 21 |
+
self._config = SQLiteStorage.get_run_config(
|
| 22 |
+
self.project, self.name, run_id=self.id
|
| 23 |
+
)
|
| 24 |
+
return self._config
|
| 25 |
+
|
| 26 |
+
def alerts(self, level: str | None = None, since: str | None = None) -> list[dict]:
|
| 27 |
+
return SQLiteStorage.get_alerts(
|
| 28 |
+
self.project, run_name=self.name, run_id=self.id, level=level, since=since
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
def delete(self) -> bool:
|
| 32 |
+
return SQLiteStorage.delete_run(self.project, self.name, run_id=self.id)
|
| 33 |
+
|
| 34 |
+
def move(self, new_project: str) -> bool:
|
| 35 |
+
success = SQLiteStorage.move_run(
|
| 36 |
+
self.project, self.name, new_project, run_id=self.id
|
| 37 |
+
)
|
| 38 |
+
if success:
|
| 39 |
+
self.project = new_project
|
| 40 |
+
return success
|
| 41 |
+
|
| 42 |
+
def rename(self, new_name: str) -> "Run":
|
| 43 |
+
SQLiteStorage.rename_run(self.project, self.name, new_name, run_id=self.id)
|
| 44 |
+
self.name = new_name
|
| 45 |
+
return self
|
| 46 |
+
|
| 47 |
+
def __repr__(self) -> str:
|
| 48 |
+
return f"<Run {self.name} in project {self.project}>"
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class Runs:
|
| 52 |
+
def __init__(self, project: str):
|
| 53 |
+
self.project = project
|
| 54 |
+
self._runs = None
|
| 55 |
+
|
| 56 |
+
def _load_runs(self):
|
| 57 |
+
if self._runs is None:
|
| 58 |
+
records = SQLiteStorage.get_run_records(self.project)
|
| 59 |
+
self._runs = [
|
| 60 |
+
Run(
|
| 61 |
+
self.project,
|
| 62 |
+
str(record["name"]),
|
| 63 |
+
run_id=str(record["id"]) if record["id"] is not None else None,
|
| 64 |
+
)
|
| 65 |
+
for record in records
|
| 66 |
+
]
|
| 67 |
+
|
| 68 |
+
def __iter__(self) -> Iterator[Run]:
|
| 69 |
+
self._load_runs()
|
| 70 |
+
return iter(self._runs)
|
| 71 |
+
|
| 72 |
+
def __getitem__(self, index: int) -> Run:
|
| 73 |
+
self._load_runs()
|
| 74 |
+
return self._runs[index]
|
| 75 |
+
|
| 76 |
+
def __len__(self) -> int:
|
| 77 |
+
self._load_runs()
|
| 78 |
+
return len(self._runs)
|
| 79 |
+
|
| 80 |
+
def __repr__(self) -> str:
|
| 81 |
+
self._load_runs()
|
| 82 |
+
return f"<Runs project={self.project} count={len(self._runs)}>"
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class Api:
|
| 86 |
+
def runs(self, project: str) -> Runs:
|
| 87 |
+
if not SQLiteStorage.get_project_db_path(project).exists():
|
| 88 |
+
raise ValueError(f"Project '{project}' does not exist")
|
| 89 |
+
return Runs(project)
|
| 90 |
+
|
| 91 |
+
def alerts(
|
| 92 |
+
self,
|
| 93 |
+
project: str,
|
| 94 |
+
run: str | None = None,
|
| 95 |
+
level: str | None = None,
|
| 96 |
+
since: str | None = None,
|
| 97 |
+
) -> list[dict]:
|
| 98 |
+
if not SQLiteStorage.get_project_db_path(project).exists():
|
| 99 |
+
raise ValueError(f"Project '{project}' does not exist")
|
| 100 |
+
return SQLiteStorage.get_alerts(project, run_name=run, level=level, since=since)
|
| 101 |
+
|
| 102 |
+
def create_registry(
|
| 103 |
+
self,
|
| 104 |
+
name: str,
|
| 105 |
+
description: str | None = None,
|
| 106 |
+
bucket_id: str | None = None,
|
| 107 |
+
) -> Registry:
|
| 108 |
+
"""Create a new registry and return a handle on it.
|
| 109 |
+
|
| 110 |
+
Raises `ValueError` if a registry with this name already exists.
|
| 111 |
+
Registries are never created implicitly: linking into a registry
|
| 112 |
+
that does not exist raises an error.
|
| 113 |
+
|
| 114 |
+
Args:
|
| 115 |
+
name (`str`):
|
| 116 |
+
Registry name, e.g. `"models"`. Must match
|
| 117 |
+
`^[A-Za-z0-9_-]+$`.
|
| 118 |
+
description (`str`, *optional*):
|
| 119 |
+
Human-readable description of the registry.
|
| 120 |
+
bucket_id (`str`, *optional*):
|
| 121 |
+
Hugging Face bucket to hold the registry, e.g.
|
| 122 |
+
`"my-org/models-registry"`. The bucket is created (private) if
|
| 123 |
+
it does not exist. A bucket-backed registry is reachable from
|
| 124 |
+
any machine with access to it, including runs that log to a
|
| 125 |
+
Space. Omit for a local registry; defaults to
|
| 126 |
+
`TRACKIO_REGISTRY_BUCKET_ID` when that is set.
|
| 127 |
+
|
| 128 |
+
Returns:
|
| 129 |
+
A [`Registry`] handle on the new registry.
|
| 130 |
+
"""
|
| 131 |
+
registry = Registry(name, bucket_id=bucket_id)
|
| 132 |
+
registry._storage.create_registry(name, description=description)
|
| 133 |
+
return registry
|
| 134 |
+
|
| 135 |
+
def registry(self, name: str, bucket_id: str | None = None) -> Registry:
|
| 136 |
+
"""Fetch a handle on an existing registry.
|
| 137 |
+
|
| 138 |
+
Pass `bucket_id` (or set `TRACKIO_REGISTRY_BUCKET_ID`) for a
|
| 139 |
+
bucket-backed registry. Raises `ValueError` if no registry with this
|
| 140 |
+
name exists."""
|
| 141 |
+
registry = Registry(name, bucket_id=bucket_id)
|
| 142 |
+
if not registry._storage.registry_exists(name):
|
| 143 |
+
where = (
|
| 144 |
+
""
|
| 145 |
+
if registry.bucket_id is None
|
| 146 |
+
else f" in bucket '{registry.bucket_id}'"
|
| 147 |
+
)
|
| 148 |
+
raise ValueError(f"Registry '{name}' does not exist{where}")
|
| 149 |
+
return registry
|
trackio/apple_gpu.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import platform
|
| 2 |
+
import subprocess
|
| 3 |
+
import sys
|
| 4 |
+
import threading
|
| 5 |
+
import warnings
|
| 6 |
+
from typing import TYPE_CHECKING, Any
|
| 7 |
+
|
| 8 |
+
if TYPE_CHECKING:
|
| 9 |
+
from trackio.run import Run
|
| 10 |
+
|
| 11 |
+
psutil: Any = None
|
| 12 |
+
PSUTIL_AVAILABLE = False
|
| 13 |
+
_monitor_lock = threading.Lock()
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _ensure_psutil():
|
| 17 |
+
global PSUTIL_AVAILABLE, psutil
|
| 18 |
+
if PSUTIL_AVAILABLE:
|
| 19 |
+
return psutil
|
| 20 |
+
try:
|
| 21 |
+
import psutil as _psutil
|
| 22 |
+
|
| 23 |
+
psutil = _psutil
|
| 24 |
+
PSUTIL_AVAILABLE = True
|
| 25 |
+
return psutil
|
| 26 |
+
except ImportError:
|
| 27 |
+
raise ImportError(
|
| 28 |
+
"psutil is required for Apple Silicon monitoring. "
|
| 29 |
+
"Install it with: pip install psutil"
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def is_apple_silicon() -> bool:
|
| 34 |
+
"""Check if running on Apple Silicon (M1/M2/M3/M4)."""
|
| 35 |
+
if platform.system() != "Darwin":
|
| 36 |
+
return False
|
| 37 |
+
|
| 38 |
+
try:
|
| 39 |
+
result = subprocess.run(
|
| 40 |
+
["sysctl", "-n", "machdep.cpu.brand_string"],
|
| 41 |
+
capture_output=True,
|
| 42 |
+
text=True,
|
| 43 |
+
timeout=1,
|
| 44 |
+
)
|
| 45 |
+
cpu_brand = result.stdout.strip()
|
| 46 |
+
return "Apple" in cpu_brand
|
| 47 |
+
except Exception:
|
| 48 |
+
return False
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def get_gpu_info() -> dict[str, Any]:
|
| 52 |
+
"""Get Apple GPU information using ioreg."""
|
| 53 |
+
try:
|
| 54 |
+
result = subprocess.run(
|
| 55 |
+
["ioreg", "-r", "-d", "1", "-w", "0", "-c", "IOAccelerator"],
|
| 56 |
+
capture_output=True,
|
| 57 |
+
text=True,
|
| 58 |
+
timeout=2,
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
if result.returncode == 0 and result.stdout:
|
| 62 |
+
lines = result.stdout.strip().split("\n")
|
| 63 |
+
for line in lines:
|
| 64 |
+
if "IOAccelerator" in line and "class" in line:
|
| 65 |
+
return {"detected": True, "type": "Apple GPU"}
|
| 66 |
+
else:
|
| 67 |
+
print("Error collecting Apple GPU info. ioreg stdout was:", file=sys.stderr)
|
| 68 |
+
print(result.stdout, file=sys.stderr)
|
| 69 |
+
print("ioreg stderr was:", file=sys.stderr)
|
| 70 |
+
print(result.stderr, file=sys.stderr)
|
| 71 |
+
|
| 72 |
+
result = subprocess.run(
|
| 73 |
+
["system_profiler", "SPDisplaysDataType"],
|
| 74 |
+
capture_output=True,
|
| 75 |
+
text=True,
|
| 76 |
+
timeout=3,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
if result.returncode == 0 and "Apple" in result.stdout:
|
| 80 |
+
for line in result.stdout.split("\n"):
|
| 81 |
+
if "Chipset Model:" in line:
|
| 82 |
+
model = line.split(":")[-1].strip()
|
| 83 |
+
return {"detected": True, "type": model}
|
| 84 |
+
|
| 85 |
+
except Exception:
|
| 86 |
+
pass
|
| 87 |
+
|
| 88 |
+
return {"detected": False}
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def apple_gpu_available() -> bool:
|
| 92 |
+
"""
|
| 93 |
+
Check if Apple GPU monitoring is available.
|
| 94 |
+
|
| 95 |
+
Returns True if running on Apple Silicon (M-series chips) and psutil is installed.
|
| 96 |
+
"""
|
| 97 |
+
try:
|
| 98 |
+
_ensure_psutil()
|
| 99 |
+
return is_apple_silicon()
|
| 100 |
+
except ImportError:
|
| 101 |
+
return False
|
| 102 |
+
except Exception:
|
| 103 |
+
return False
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def collect_apple_metrics(include_cpu_metrics: bool = True) -> dict:
|
| 107 |
+
"""
|
| 108 |
+
Collect system metrics for Apple Silicon.
|
| 109 |
+
|
| 110 |
+
Returns:
|
| 111 |
+
Dictionary of system metrics including CPU, memory, and GPU info.
|
| 112 |
+
"""
|
| 113 |
+
metrics = {}
|
| 114 |
+
|
| 115 |
+
if include_cpu_metrics:
|
| 116 |
+
from trackio.cpu import collect_cpu_metrics
|
| 117 |
+
|
| 118 |
+
metrics.update(collect_cpu_metrics(include_static=True))
|
| 119 |
+
|
| 120 |
+
gpu_info = get_gpu_info()
|
| 121 |
+
if gpu_info.get("detected"):
|
| 122 |
+
metrics["gpu/detected"] = 1
|
| 123 |
+
|
| 124 |
+
return metrics
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
class AppleGpuMonitor:
|
| 128 |
+
def __init__(
|
| 129 |
+
self, run: "Run", interval: float = 10.0, include_cpu_metrics: bool = True
|
| 130 |
+
):
|
| 131 |
+
self._run = run
|
| 132 |
+
self._interval = interval
|
| 133 |
+
self._include_cpu_metrics = include_cpu_metrics
|
| 134 |
+
self._stop_flag = threading.Event()
|
| 135 |
+
self._thread: "threading.Thread | None" = None
|
| 136 |
+
|
| 137 |
+
def start(self):
|
| 138 |
+
if not is_apple_silicon():
|
| 139 |
+
warnings.warn(
|
| 140 |
+
"auto_log_gpu=True but not running on Apple Silicon. "
|
| 141 |
+
"Apple GPU logging disabled."
|
| 142 |
+
)
|
| 143 |
+
return
|
| 144 |
+
|
| 145 |
+
if not PSUTIL_AVAILABLE:
|
| 146 |
+
try:
|
| 147 |
+
_ensure_psutil()
|
| 148 |
+
except ImportError:
|
| 149 |
+
warnings.warn(
|
| 150 |
+
"auto_log_gpu=True but psutil not installed. "
|
| 151 |
+
"Install with: pip install psutil"
|
| 152 |
+
)
|
| 153 |
+
return
|
| 154 |
+
|
| 155 |
+
self._thread = threading.Thread(target=self._monitor_loop, daemon=True)
|
| 156 |
+
self._thread.start()
|
| 157 |
+
|
| 158 |
+
def stop(self):
|
| 159 |
+
self._stop_flag.set()
|
| 160 |
+
if self._thread is not None:
|
| 161 |
+
self._thread.join(timeout=2.0)
|
| 162 |
+
|
| 163 |
+
def _monitor_loop(self):
|
| 164 |
+
while not self._stop_flag.is_set():
|
| 165 |
+
try:
|
| 166 |
+
metrics = collect_apple_metrics(
|
| 167 |
+
include_cpu_metrics=self._include_cpu_metrics
|
| 168 |
+
)
|
| 169 |
+
if metrics:
|
| 170 |
+
self._run.log_system(metrics)
|
| 171 |
+
except Exception:
|
| 172 |
+
pass
|
| 173 |
+
|
| 174 |
+
self._stop_flag.wait(timeout=self._interval)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def log_apple_gpu(run: "Run | None" = None) -> dict:
|
| 178 |
+
"""
|
| 179 |
+
Log Apple Silicon system metrics to the current or specified run.
|
| 180 |
+
|
| 181 |
+
Args:
|
| 182 |
+
run: Optional Run instance. If None, uses current run from context.
|
| 183 |
+
|
| 184 |
+
Returns:
|
| 185 |
+
dict: The system metrics that were logged.
|
| 186 |
+
|
| 187 |
+
Example:
|
| 188 |
+
```python
|
| 189 |
+
import trackio
|
| 190 |
+
|
| 191 |
+
run = trackio.init(project="my-project")
|
| 192 |
+
trackio.log({"loss": 0.5})
|
| 193 |
+
trackio.log_apple_gpu()
|
| 194 |
+
```
|
| 195 |
+
"""
|
| 196 |
+
from trackio import context_vars
|
| 197 |
+
|
| 198 |
+
if run is None:
|
| 199 |
+
run = context_vars.current_run.get()
|
| 200 |
+
if run is None:
|
| 201 |
+
raise RuntimeError("Call trackio.init() before trackio.log_apple_gpu().")
|
| 202 |
+
|
| 203 |
+
metrics = collect_apple_metrics()
|
| 204 |
+
if metrics:
|
| 205 |
+
run.log_system(metrics)
|
| 206 |
+
return metrics
|
trackio/artifact.py
ADDED
|
@@ -0,0 +1,774 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
import httpx
|
| 6 |
+
from huggingface_hub.utils import get_token
|
| 7 |
+
|
| 8 |
+
from trackio import cas, references, utils
|
| 9 |
+
from trackio.registry import registry_backend
|
| 10 |
+
from trackio.registry_storage import (
|
| 11 |
+
parse_collection_target,
|
| 12 |
+
registry_project_name,
|
| 13 |
+
)
|
| 14 |
+
from trackio.remote_client import _merge_client_headers, _resolve_src_url
|
| 15 |
+
from trackio.typehints import ETag, Manifest, ManifestEntry, Sha256Digest, URIStr
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _materialize(blob: Path, dst: Path, size: int) -> None:
|
| 19 |
+
"""Copy `blob` to `dst` unless `dst` already matches the blob's size and
|
| 20 |
+
mtime.
|
| 21 |
+
"""
|
| 22 |
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
| 23 |
+
blob_stat = blob.stat()
|
| 24 |
+
if dst.is_file():
|
| 25 |
+
dst_stat = dst.stat()
|
| 26 |
+
if dst_stat.st_size == size and dst_stat.st_mtime_ns == blob_stat.st_mtime_ns:
|
| 27 |
+
return
|
| 28 |
+
partial = dst.parent / cas.partial_blob_name(dst.name)
|
| 29 |
+
try:
|
| 30 |
+
shutil.copyfile(blob, partial)
|
| 31 |
+
os.utime(partial, ns=(blob_stat.st_atime_ns, blob_stat.st_mtime_ns))
|
| 32 |
+
os.replace(partial, dst)
|
| 33 |
+
except Exception:
|
| 34 |
+
partial.unlink(missing_ok=True)
|
| 35 |
+
raise
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _fetch_blob_from_remote(
|
| 39 |
+
remote_source: dict,
|
| 40 |
+
project: str,
|
| 41 |
+
digest: Sha256Digest,
|
| 42 |
+
target_path: Path,
|
| 43 |
+
) -> None:
|
| 44 |
+
space_id = remote_source.get("space_id")
|
| 45 |
+
src = space_id or remote_source.get("server_base_url")
|
| 46 |
+
if not src:
|
| 47 |
+
raise RuntimeError(
|
| 48 |
+
"Artifact has _remote_source set but neither space_id nor "
|
| 49 |
+
"server_base_url is populated."
|
| 50 |
+
)
|
| 51 |
+
base_url = _resolve_src_url(src).rstrip("/")
|
| 52 |
+
url = f"{base_url}/artifact_blob/{utils.canonical_project_name(project)}/{digest}"
|
| 53 |
+
headers = _merge_client_headers(
|
| 54 |
+
get_token() if space_id else None,
|
| 55 |
+
remote_source.get("write_token"),
|
| 56 |
+
)
|
| 57 |
+
with httpx.stream(
|
| 58 |
+
"GET",
|
| 59 |
+
url,
|
| 60 |
+
headers=headers,
|
| 61 |
+
timeout=httpx.Timeout(connect=10.0, read=300.0, write=10.0, pool=10.0),
|
| 62 |
+
) as response:
|
| 63 |
+
if response.status_code == 404:
|
| 64 |
+
raise FileNotFoundError(
|
| 65 |
+
f"Artifact blob {digest} not available on remote at {url}"
|
| 66 |
+
)
|
| 67 |
+
response.raise_for_status()
|
| 68 |
+
cas.stage_blob_from_chunks(
|
| 69 |
+
response.iter_bytes(),
|
| 70 |
+
claimed_digest=digest,
|
| 71 |
+
target_path=target_path,
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _materialize_reference(
|
| 76 |
+
uri: str, scheme: str, dest: Path, digest: Sha256Digest | ETag | URIStr
|
| 77 |
+
) -> None:
|
| 78 |
+
"""Fetch the object at `uri` into `dest`, idempotently and atomically.
|
| 79 |
+
|
| 80 |
+
If `dest` already exists it is left untouched and nothing is fetched:
|
| 81 |
+
reference digests are opaque, so a file already present is assumed current
|
| 82 |
+
(mirroring the size/mtime idempotence of `_materialize`), letting a repeated
|
| 83 |
+
`download()` skip references already on disk.
|
| 84 |
+
|
| 85 |
+
Otherwise, when a real checksum was recorded, the source is re-probed and
|
| 86 |
+
compared. When the digest doesn't exist, or when it cannot be re-probed,
|
| 87 |
+
the comparison is skipped and the artifact is still materialized.
|
| 88 |
+
"""
|
| 89 |
+
if dest.is_file():
|
| 90 |
+
return
|
| 91 |
+
if len(digest) > 0 and digest != uri:
|
| 92 |
+
try:
|
| 93 |
+
resolved = references.resolve_reference(
|
| 94 |
+
uri, scheme, checksum=True, max_objects=1
|
| 95 |
+
)[0]
|
| 96 |
+
except Exception:
|
| 97 |
+
resolved = None
|
| 98 |
+
current_digest = resolved.digest if resolved is not None else None
|
| 99 |
+
if current_digest is not None and current_digest != digest:
|
| 100 |
+
utils._emit_nonfatal_warning(
|
| 101 |
+
f"Reference {uri!r} may have changed since it was logged "
|
| 102 |
+
f"(recorded digest {digest}, current {current_digest})."
|
| 103 |
+
)
|
| 104 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 105 |
+
partial = dest.parent / cas.partial_blob_name(dest.name)
|
| 106 |
+
try:
|
| 107 |
+
references.fetch_reference(uri, scheme, partial)
|
| 108 |
+
os.replace(partial, dest)
|
| 109 |
+
except Exception:
|
| 110 |
+
partial.unlink(missing_ok=True)
|
| 111 |
+
raise
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
class Artifact:
|
| 115 |
+
"""
|
| 116 |
+
A versioned, named bundle of files and references attached to a project.
|
| 117 |
+
|
| 118 |
+
Construct an `Artifact`, add files to it with `add_file` / `add_dir`
|
| 119 |
+
(bytes staged into content-addressed storage) or `add_reference` (an
|
| 120 |
+
external URI recorded without copying any bytes), then persist it with
|
| 121 |
+
`trackio.log_artifact`. Logging freezes the artifact and populates its
|
| 122 |
+
read-only `version`, `aliases`, `size`, and `manifest`.
|
| 123 |
+
|
| 124 |
+
Args:
|
| 125 |
+
name (`str`):
|
| 126 |
+
Artifact name, unique within the project. Must match
|
| 127 |
+
`^[A-Za-z0-9._-]+$` (letters, digits, dot, underscore, hyphen).
|
| 128 |
+
type (`str`):
|
| 129 |
+
Free-form category such as `"model"` or `"dataset"`, used to group
|
| 130 |
+
artifacts.
|
| 131 |
+
description (`str`, *optional*):
|
| 132 |
+
Human-readable description of the artifact.
|
| 133 |
+
metadata (`dict`, *optional*):
|
| 134 |
+
Arbitrary JSON-serializable metadata stored alongside the version.
|
| 135 |
+
"""
|
| 136 |
+
|
| 137 |
+
def __init__(
|
| 138 |
+
self,
|
| 139 |
+
name: str,
|
| 140 |
+
type: str,
|
| 141 |
+
description: str | None = None,
|
| 142 |
+
metadata: dict | None = None,
|
| 143 |
+
):
|
| 144 |
+
cas.validate_artifact_name(name)
|
| 145 |
+
self._name = name
|
| 146 |
+
self._type = type
|
| 147 |
+
self._description = description
|
| 148 |
+
self._metadata: dict = dict(metadata) if metadata else {}
|
| 149 |
+
self._pending_files: list[tuple[Path, str]] = []
|
| 150 |
+
self._pending_references: list[
|
| 151 |
+
tuple[str, str, Sha256Digest | ETag | URIStr, int]
|
| 152 |
+
] = []
|
| 153 |
+
self._logged = False
|
| 154 |
+
self._version: int | None = None
|
| 155 |
+
self._aliases: tuple[str, ...] = ()
|
| 156 |
+
self._size: int | None = None
|
| 157 |
+
self._manifest: Manifest | None = None
|
| 158 |
+
self._manifest_digest: Sha256Digest | None = None
|
| 159 |
+
self._project: str | None = None
|
| 160 |
+
self._remote_source: dict | None = None
|
| 161 |
+
self._registry: str | None = None
|
| 162 |
+
self._registry_bucket_id: str | None = None
|
| 163 |
+
self._source_project: str | None = None
|
| 164 |
+
self._source_artifact: str | None = None
|
| 165 |
+
self._source_version: int | None = None
|
| 166 |
+
|
| 167 |
+
@property
|
| 168 |
+
def name(self) -> str:
|
| 169 |
+
"""Artifact name, unique within its project."""
|
| 170 |
+
return self._name
|
| 171 |
+
|
| 172 |
+
@property
|
| 173 |
+
def type(self) -> str:
|
| 174 |
+
"""Free-form category such as `"model"` or `"dataset"`."""
|
| 175 |
+
return self._type
|
| 176 |
+
|
| 177 |
+
@property
|
| 178 |
+
def description(self) -> str | None:
|
| 179 |
+
"""Human-readable description, or None if unset."""
|
| 180 |
+
return self._description
|
| 181 |
+
|
| 182 |
+
@description.setter
|
| 183 |
+
def description(self, value: str | None) -> None:
|
| 184 |
+
self._description = value
|
| 185 |
+
|
| 186 |
+
@property
|
| 187 |
+
def metadata(self) -> dict:
|
| 188 |
+
"""Live, mutable metadata dict; assignments like `art.metadata["k"] = v`
|
| 189 |
+
made before logging are preserved."""
|
| 190 |
+
return self._metadata
|
| 191 |
+
|
| 192 |
+
@metadata.setter
|
| 193 |
+
def metadata(self, value: dict | None) -> None:
|
| 194 |
+
self._metadata = dict(value) if value else {}
|
| 195 |
+
|
| 196 |
+
@property
|
| 197 |
+
def version(self) -> str | None:
|
| 198 |
+
"""Version string in `"v<N>"` form (e.g. `"v3"`), or None if the
|
| 199 |
+
artifact has not been logged or fetched yet."""
|
| 200 |
+
return None if self._version is None else f"v{self._version}"
|
| 201 |
+
|
| 202 |
+
@property
|
| 203 |
+
def aliases(self) -> tuple[str, ...]:
|
| 204 |
+
"""Aliases pointing at this version, e.g. `("latest", "prod")`."""
|
| 205 |
+
return self._aliases
|
| 206 |
+
|
| 207 |
+
@property
|
| 208 |
+
def size(self) -> int | None:
|
| 209 |
+
"""Total size in bytes of the artifact's files plus any referenced data
|
| 210 |
+
whose size is known, or None if not yet logged."""
|
| 211 |
+
return self._size
|
| 212 |
+
|
| 213 |
+
@property
|
| 214 |
+
def manifest(self) -> Manifest | None:
|
| 215 |
+
"""List of manifest entries describing the artifact's contents, or None
|
| 216 |
+
if not yet logged or fetched. File entries carry `path`, `digest`, and
|
| 217 |
+
`size`; reference entries carry `path`, `ref` (a URI), `size`, and a
|
| 218 |
+
`digest` — a sha256 for local/LFS references, the provider's opaque ETag
|
| 219 |
+
for HTTP or object-store references, or the URI itself when no checksum
|
| 220 |
+
is available. Returns a fresh copy on each access."""
|
| 221 |
+
return None if self._manifest is None else [dict(e) for e in self._manifest]
|
| 222 |
+
|
| 223 |
+
@property
|
| 224 |
+
def manifest_digest(self) -> Sha256Digest | None:
|
| 225 |
+
"""Digest over the manifest (file paths plus their content), used to
|
| 226 |
+
de-duplicate re-logged content; None until logged or fetched."""
|
| 227 |
+
return self._manifest_digest
|
| 228 |
+
|
| 229 |
+
@property
|
| 230 |
+
def digest(self) -> Sha256Digest | None:
|
| 231 |
+
"""Alias of `manifest_digest`."""
|
| 232 |
+
return self._manifest_digest
|
| 233 |
+
|
| 234 |
+
@property
|
| 235 |
+
def qualified_name(self) -> str:
|
| 236 |
+
"""`"<project>/<name>:v<N>"` for a logged or fetched artifact."""
|
| 237 |
+
if self._project is None or self._version is None:
|
| 238 |
+
raise RuntimeError(
|
| 239 |
+
"Artifact has no qualified name until it is logged or fetched."
|
| 240 |
+
)
|
| 241 |
+
return f"{self._project}/{self._name}:v{self._version}"
|
| 242 |
+
|
| 243 |
+
@property
|
| 244 |
+
def project(self) -> str | None:
|
| 245 |
+
"""Project the artifact belongs to, or None until logged or fetched."""
|
| 246 |
+
return self._project
|
| 247 |
+
|
| 248 |
+
@property
|
| 249 |
+
def is_link(self) -> bool:
|
| 250 |
+
"""True if this artifact is a registry link, i.e. the pointer
|
| 251 |
+
returned by `Run.link_artifact`, rather than a source artifact
|
| 252 |
+
version."""
|
| 253 |
+
return self._registry is not None
|
| 254 |
+
|
| 255 |
+
@property
|
| 256 |
+
def source_name(self) -> str | None:
|
| 257 |
+
"""Name of the source artifact version this artifact's content comes
|
| 258 |
+
from. Equal to `name` unless this artifact is a registry link."""
|
| 259 |
+
return self._source_artifact if self.is_link else self._name
|
| 260 |
+
|
| 261 |
+
@property
|
| 262 |
+
def source_version(self) -> str | None:
|
| 263 |
+
"""Version string (`"v<N>"`) of the source artifact version. Equal to
|
| 264 |
+
`version` unless this artifact is a registry link."""
|
| 265 |
+
if self.is_link:
|
| 266 |
+
return f"v{self._source_version}"
|
| 267 |
+
return self.version
|
| 268 |
+
|
| 269 |
+
@property
|
| 270 |
+
def source_project(self) -> str | None:
|
| 271 |
+
"""Project the source artifact version was logged in. Equal to
|
| 272 |
+
`project` unless this artifact is a registry link."""
|
| 273 |
+
return self._source_project if self.is_link else self._project
|
| 274 |
+
|
| 275 |
+
@property
|
| 276 |
+
def source_qualified_name(self) -> str:
|
| 277 |
+
"""`"<project>/<name>:v<N>"` of the source artifact version. Equal to
|
| 278 |
+
`qualified_name` unless this artifact is a registry link."""
|
| 279 |
+
if self.is_link:
|
| 280 |
+
return (
|
| 281 |
+
f"{self._source_project}/{self._source_artifact}"
|
| 282 |
+
f":v{self._source_version}"
|
| 283 |
+
)
|
| 284 |
+
return self.qualified_name
|
| 285 |
+
|
| 286 |
+
def wait(self, timeout: int | None = None) -> "Artifact":
|
| 287 |
+
"""No-op: trackio logs artifacts synchronously, so the artifact is
|
| 288 |
+
already committed by the time `log_artifact` returns."""
|
| 289 |
+
if not self._logged:
|
| 290 |
+
raise RuntimeError(
|
| 291 |
+
"Cannot wait on an Artifact that has not been logged; "
|
| 292 |
+
"call log_artifact first."
|
| 293 |
+
)
|
| 294 |
+
return self
|
| 295 |
+
|
| 296 |
+
def add_file(self, local_path: str | Path, name: str | None = None) -> None:
|
| 297 |
+
"""Stage a single file for inclusion in the artifact.
|
| 298 |
+
|
| 299 |
+
Args:
|
| 300 |
+
local_path (`str` or `Path`):
|
| 301 |
+
Path to an existing regular file to add.
|
| 302 |
+
name (`str`, *optional*):
|
| 303 |
+
Logical path the file is stored under inside the artifact.
|
| 304 |
+
Defaults to the file's basename.
|
| 305 |
+
"""
|
| 306 |
+
if self._logged:
|
| 307 |
+
raise RuntimeError(
|
| 308 |
+
"Cannot add files to an Artifact that has already been logged or fetched."
|
| 309 |
+
)
|
| 310 |
+
src = Path(local_path).resolve()
|
| 311 |
+
if not src.is_file():
|
| 312 |
+
raise ValueError(f"Not a regular file: {local_path}")
|
| 313 |
+
logical = cas.validate_logical_path(name if name is not None else src.name)
|
| 314 |
+
self._pending_files.append((src, logical))
|
| 315 |
+
|
| 316 |
+
def add_dir(self, local_dir: str | Path, name: str | None = None) -> None:
|
| 317 |
+
"""Stage every regular file under a directory, recursively.
|
| 318 |
+
|
| 319 |
+
Symlinks are skipped.
|
| 320 |
+
|
| 321 |
+
Args:
|
| 322 |
+
local_dir (`str` or `Path`):
|
| 323 |
+
Path to an existing directory whose files are added.
|
| 324 |
+
name (`str`, *optional*):
|
| 325 |
+
Logical path prefix for the added files. Defaults to no prefix,
|
| 326 |
+
so files keep their paths relative to `local_dir`.
|
| 327 |
+
"""
|
| 328 |
+
if self._logged:
|
| 329 |
+
raise RuntimeError(
|
| 330 |
+
"Cannot add files to an Artifact that has already been logged or fetched."
|
| 331 |
+
)
|
| 332 |
+
root = Path(local_dir).resolve()
|
| 333 |
+
if not root.is_dir():
|
| 334 |
+
raise ValueError(f"Not a directory: {local_dir}")
|
| 335 |
+
prefix = cas.validate_logical_path(name.rstrip("/")) + "/" if name else ""
|
| 336 |
+
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
| 337 |
+
dirnames.sort()
|
| 338 |
+
for filename in sorted(filenames):
|
| 339 |
+
entry = Path(dirpath) / filename
|
| 340 |
+
if entry.is_symlink() or not entry.is_file():
|
| 341 |
+
continue
|
| 342 |
+
rel = entry.relative_to(root).as_posix()
|
| 343 |
+
logical = cas.validate_logical_path(prefix + rel)
|
| 344 |
+
self._pending_files.append((entry, logical))
|
| 345 |
+
|
| 346 |
+
def add_reference(
|
| 347 |
+
self,
|
| 348 |
+
uri: str,
|
| 349 |
+
name: str | None = None,
|
| 350 |
+
checksum: bool = True,
|
| 351 |
+
max_objects: int | None = None,
|
| 352 |
+
) -> None:
|
| 353 |
+
"""Stage a reference to external data, without copying any bytes.
|
| 354 |
+
|
| 355 |
+
Unlike `add_file`/`add_dir`, a reference records the data's URI (plus,
|
| 356 |
+
when available, a size and checksum) *without staging the bytes into
|
| 357 |
+
the content-addressed store*. This is how you version and track
|
| 358 |
+
lineage over large or already-durably-stored datasets that must not be
|
| 359 |
+
duplicated.
|
| 360 |
+
|
| 361 |
+
A URI that denotes a directory (`file://`, `hf://`, or a prefix
|
| 362 |
+
expanded by a custom handler) is expanded into one entry per object,
|
| 363 |
+
capped by `max_objects`. Reference bytes are never uploaded to a Space
|
| 364 |
+
or bucket.
|
| 365 |
+
|
| 366 |
+
The URI is stored verbatim in the artifact manifest, and manifests may
|
| 367 |
+
be synced to a shared Hugging Face Dataset — so a signed URI triggers a
|
| 368 |
+
warning for security reasons; please prefer the object's canonical
|
| 369 |
+
unsigned URI (e.g. `s3://bucket/key`).
|
| 370 |
+
|
| 371 |
+
Args:
|
| 372 |
+
uri (`str`):
|
| 373 |
+
Location of the referenced data, percent-encoded (build local
|
| 374 |
+
URIs with `pathlib.Path.as_uri()`). `file://`, `http(s)://`,
|
| 375 |
+
and `hf://` are resolved and fetched natively; any other scheme
|
| 376 |
+
(`s3://`, `gs://`, Azure blob URLs, ...) requires a custom
|
| 377 |
+
`ReferenceHandler` registered via
|
| 378 |
+
`trackio.register_reference_handler` — see the artifacts
|
| 379 |
+
documentation for complete examples — and raises otherwise.
|
| 380 |
+
name (`str`, *optional*):
|
| 381 |
+
Logical path the reference is stored under. Defaults to the last
|
| 382 |
+
segment of `uri`; for an expanded prefix (ending with /) it is the
|
| 383 |
+
prefix under which each object's relative path is nested.
|
| 384 |
+
checksum (`bool`, *optional*, defaults to `True`):
|
| 385 |
+
When `True`, probes the source to derive its size and a
|
| 386 |
+
checksum (and to expand a directory/prefix): `file://` is
|
| 387 |
+
stream-hashed to a sha256, `hf://` uses its LFS sha256 or git
|
| 388 |
+
blob id, and `http(s)://` uses the server `ETag`. This may read
|
| 389 |
+
the filesystem or make network requests. When no checksum can
|
| 390 |
+
be obtained for a *remote* source (e.g. no ETag), the URI
|
| 391 |
+
itself is used as the `digest` and a warning is emitted. A
|
| 392 |
+
local `file://` path that does not exist instead raises. Pass
|
| 393 |
+
`checksum=False` to skip remote probing entirely.
|
| 394 |
+
max_objects (`int`, *optional*):
|
| 395 |
+
Cap on how many objects a directory/prefix expands into (default
|
| 396 |
+
10,000); exceeding it raises.
|
| 397 |
+
"""
|
| 398 |
+
if self._logged:
|
| 399 |
+
raise RuntimeError(
|
| 400 |
+
"Cannot add references to an Artifact that has already been "
|
| 401 |
+
"logged or fetched."
|
| 402 |
+
)
|
| 403 |
+
scheme, normalized = references.validate_reference_uri(uri)
|
| 404 |
+
if references.looks_signed(normalized):
|
| 405 |
+
utils._emit_nonfatal_warning(
|
| 406 |
+
f"Reference {uri!r} looks like a signed URL, and the full URI "
|
| 407 |
+
"(embedded credential included) is stored in the artifact "
|
| 408 |
+
"manifest, which may be synced to a shared HF Dataset. Prefer "
|
| 409 |
+
"the object's canonical unsigned URI (e.g. s3://bucket/key or "
|
| 410 |
+
"the URL without its signed query string)."
|
| 411 |
+
)
|
| 412 |
+
cap = references.DEFAULT_MAX_OBJECTS if max_objects is None else max_objects
|
| 413 |
+
resolved = references.resolve_reference(normalized, scheme, checksum, cap)
|
| 414 |
+
if not resolved:
|
| 415 |
+
raise ValueError(f"Reference URI {uri!r} matched no objects.")
|
| 416 |
+
used_uri_fallback = False
|
| 417 |
+
name_prefix = f"{name.rstrip('/')}/" if name is not None else ""
|
| 418 |
+
for entry in resolved:
|
| 419 |
+
if entry.relkey is None:
|
| 420 |
+
logical = self._reference_logical_name(name, entry.uri, uri)
|
| 421 |
+
else:
|
| 422 |
+
logical = cas.validate_logical_path(name_prefix + entry.relkey)
|
| 423 |
+
ref_digest: Sha256Digest | ETag | URIStr | None = entry.digest
|
| 424 |
+
if ref_digest is None:
|
| 425 |
+
ref_digest = URIStr(entry.uri)
|
| 426 |
+
used_uri_fallback = True
|
| 427 |
+
ref_size = 0 if entry.size is None else entry.size
|
| 428 |
+
self._pending_references.append((logical, entry.uri, ref_digest, ref_size))
|
| 429 |
+
if checksum and used_uri_fallback:
|
| 430 |
+
utils._emit_nonfatal_warning(
|
| 431 |
+
f"Reference {uri!r} was recorded without a content checksum, so its "
|
| 432 |
+
"version identity is its URI; re-logging changed content at the same "
|
| 433 |
+
"location will reuse the existing version instead of creating a new "
|
| 434 |
+
f"one. {references.checksum_hint(scheme, normalized)}"
|
| 435 |
+
)
|
| 436 |
+
|
| 437 |
+
@staticmethod
|
| 438 |
+
def _reference_logical_name(name: str | None, resolved_uri: str, uri: str) -> str:
|
| 439 |
+
if name is not None:
|
| 440 |
+
return cas.validate_logical_path(name)
|
| 441 |
+
derived = references.default_reference_name(resolved_uri)
|
| 442 |
+
if not derived:
|
| 443 |
+
raise ValueError(
|
| 444 |
+
f"Could not derive a name from URI {uri!r}; pass name= explicitly."
|
| 445 |
+
)
|
| 446 |
+
return cas.validate_logical_path(derived)
|
| 447 |
+
|
| 448 |
+
def _build_manifest(self, project: str) -> Manifest:
|
| 449 |
+
if not self._pending_files and not self._pending_references:
|
| 450 |
+
raise ValueError(
|
| 451 |
+
f"Artifact {self._name!r} has no files or references; "
|
| 452 |
+
"call add_file/add_dir/add_reference first."
|
| 453 |
+
)
|
| 454 |
+
cas.assert_manifest_paths_compatible(
|
| 455 |
+
[logical for _, logical in self._pending_files]
|
| 456 |
+
+ [logical for logical, *_ in self._pending_references]
|
| 457 |
+
)
|
| 458 |
+
|
| 459 |
+
entries: Manifest = []
|
| 460 |
+
for src, logical in self._pending_files:
|
| 461 |
+
digest, size = cas.stage_blob_into_project(src, project)
|
| 462 |
+
entries.append({"path": logical, "digest": digest, "size": size})
|
| 463 |
+
for logical, uri, ref_digest, ref_size in self._pending_references:
|
| 464 |
+
entry: ManifestEntry = {
|
| 465 |
+
"path": logical,
|
| 466 |
+
"ref": uri,
|
| 467 |
+
"digest": ref_digest,
|
| 468 |
+
"size": ref_size,
|
| 469 |
+
}
|
| 470 |
+
entries.append(entry)
|
| 471 |
+
return entries
|
| 472 |
+
|
| 473 |
+
def _hydrate_from_db(
|
| 474 |
+
self,
|
| 475 |
+
*,
|
| 476 |
+
project: str,
|
| 477 |
+
version: int,
|
| 478 |
+
aliases: list[str],
|
| 479 |
+
manifest: Manifest,
|
| 480 |
+
manifest_digest: Sha256Digest,
|
| 481 |
+
size_bytes: int,
|
| 482 |
+
description: str | None = None,
|
| 483 |
+
metadata: dict | None = None,
|
| 484 |
+
) -> None:
|
| 485 |
+
if description is not None:
|
| 486 |
+
self._description = description
|
| 487 |
+
if metadata is not None:
|
| 488 |
+
self._metadata = dict(metadata)
|
| 489 |
+
self._project = project
|
| 490 |
+
self._version = version
|
| 491 |
+
self._aliases = tuple(aliases)
|
| 492 |
+
self._manifest = [dict(e) for e in manifest]
|
| 493 |
+
self._manifest_digest = manifest_digest
|
| 494 |
+
self._size = size_bytes
|
| 495 |
+
self._logged = True
|
| 496 |
+
|
| 497 |
+
def download(self, root: str | Path | None = None) -> str:
|
| 498 |
+
"""Materialize the artifact's files into a local directory.
|
| 499 |
+
|
| 500 |
+
Files are copied from Trackio's content-addressed cache (and fetched
|
| 501 |
+
from the remote when missing locally), so repeated calls are cheap and
|
| 502 |
+
idempotent.
|
| 503 |
+
|
| 504 |
+
Reference entries added via `add_reference` are fetched here too: the
|
| 505 |
+
object at each reference URI is downloaded into the directory (via the
|
| 506 |
+
bundled `httpx` for `http(s)://`, `huggingface_hub` for `hf://`, and
|
| 507 |
+
the registered custom `ReferenceHandler` for any other scheme). Each
|
| 508 |
+
object is written atomically and calls are idempotent. Because reference digests
|
| 509 |
+
are opaque (a provider ETag or the URI itself), a present file is assumed
|
| 510 |
+
current and is not re-verified against the source, and a source whose
|
| 511 |
+
re-probed checksum no longer matches the recorded one is downloaded
|
| 512 |
+
anyway with a warning. Downloading can transfer a large amount of data;
|
| 513 |
+
to work with the URIs without downloading, read them from the `references`
|
| 514 |
+
property or `get_entry_uri` instead. A missing source, an unreachable
|
| 515 |
+
server, or a scheme with no registered handler raises.
|
| 516 |
+
|
| 517 |
+
Args:
|
| 518 |
+
root (`str` or `Path`, *optional*):
|
| 519 |
+
Directory to write the files into. Defaults to
|
| 520 |
+
`./.trackio/artifact-downloads/<project>/<name>_v<version>/`,
|
| 521 |
+
keyed by project so same-named artifacts from different projects
|
| 522 |
+
never collide, and by the resolved version so a moving alias
|
| 523 |
+
like `latest` never leaves behind stale files from a previous
|
| 524 |
+
version.
|
| 525 |
+
|
| 526 |
+
Returns:
|
| 527 |
+
The absolute path to the download directory, as a string.
|
| 528 |
+
"""
|
| 529 |
+
if not self._logged:
|
| 530 |
+
raise RuntimeError(
|
| 531 |
+
"Cannot download an Artifact that has not been logged or fetched."
|
| 532 |
+
)
|
| 533 |
+
if self._manifest is None or self._project is None or self._version is None:
|
| 534 |
+
raise RuntimeError("Artifact is missing manifest, project, or version.")
|
| 535 |
+
blob_project = self.source_project or self._project
|
| 536 |
+
|
| 537 |
+
if root is None:
|
| 538 |
+
project = utils.canonical_project_name(self._project)
|
| 539 |
+
root_path = (
|
| 540 |
+
Path.cwd()
|
| 541 |
+
/ ".trackio"
|
| 542 |
+
/ "artifact-downloads"
|
| 543 |
+
/ project
|
| 544 |
+
/ f"{self._name}_v{self._version}"
|
| 545 |
+
)
|
| 546 |
+
else:
|
| 547 |
+
root_path = Path(root)
|
| 548 |
+
root_path.mkdir(parents=True, exist_ok=True)
|
| 549 |
+
|
| 550 |
+
for entry in self._manifest:
|
| 551 |
+
logical = cas.validate_logical_path(entry["path"])
|
| 552 |
+
if references.is_reference_entry(entry):
|
| 553 |
+
scheme, uri = references.validate_reference_uri(entry["ref"])
|
| 554 |
+
_materialize_reference(
|
| 555 |
+
uri, scheme, root_path / logical, entry["digest"]
|
| 556 |
+
)
|
| 557 |
+
continue
|
| 558 |
+
digest = cas.validate_digest(entry["digest"])
|
| 559 |
+
blob = cas.blob_path(blob_project, digest)
|
| 560 |
+
if not blob.is_file():
|
| 561 |
+
if self._remote_source is None:
|
| 562 |
+
raise FileNotFoundError(
|
| 563 |
+
f"Artifact blob {digest} not available locally or "
|
| 564 |
+
"remotely. The producer machine may not have shipped "
|
| 565 |
+
"this blob yet."
|
| 566 |
+
)
|
| 567 |
+
_fetch_blob_from_remote(self._remote_source, blob_project, digest, blob)
|
| 568 |
+
_materialize(blob, root_path / logical, entry["size"])
|
| 569 |
+
|
| 570 |
+
return str(root_path)
|
| 571 |
+
|
| 572 |
+
def link(
|
| 573 |
+
self,
|
| 574 |
+
target_path: str,
|
| 575 |
+
aliases: list[str] | None = None,
|
| 576 |
+
bucket_id: str | None = None,
|
| 577 |
+
) -> "Artifact":
|
| 578 |
+
"""Link this artifact version into a registry collection.
|
| 579 |
+
|
| 580 |
+
A link is a pointer to this version: no files are copied. See
|
| 581 |
+
`Run.link_artifact` for the full semantics (collection versioning,
|
| 582 |
+
aliases, and the returned linked artifact).
|
| 583 |
+
|
| 584 |
+
The artifact must already be logged or fetched; unlike
|
| 585 |
+
`Run.link_artifact`, this does not log a draft artifact for you, and
|
| 586 |
+
the link records no publishing run. An artifact fetched from a Space or
|
| 587 |
+
a self-hosted server can only be linked into a bucket-backed registry,
|
| 588 |
+
since a local registry is not reachable from where its bytes live.
|
| 589 |
+
|
| 590 |
+
Args:
|
| 591 |
+
target_path (`str`):
|
| 592 |
+
The collection to link into, as
|
| 593 |
+
`"registry-<registry>/<collection>"`. The registry must
|
| 594 |
+
already exist.
|
| 595 |
+
aliases (`list[str]`, *optional*):
|
| 596 |
+
Aliases to place on the linked version. `latest` is managed
|
| 597 |
+
automatically; passing it is a no-op.
|
| 598 |
+
bucket_id (`str`, *optional*):
|
| 599 |
+
Bucket holding the registry, e.g. `"my-org/models-registry"`.
|
| 600 |
+
Omit for a local registry; defaults to
|
| 601 |
+
`TRACKIO_REGISTRY_BUCKET_ID` when that is set.
|
| 602 |
+
|
| 603 |
+
Returns:
|
| 604 |
+
The linked artifact at its registry location.
|
| 605 |
+
"""
|
| 606 |
+
if not self._logged:
|
| 607 |
+
raise RuntimeError(
|
| 608 |
+
"Cannot link an Artifact that has not been logged or fetched. "
|
| 609 |
+
"Log it first with log_artifact, or fetch a version with "
|
| 610 |
+
"use_artifact, then link the result."
|
| 611 |
+
)
|
| 612 |
+
registry, collection = parse_collection_target(target_path)
|
| 613 |
+
return self._link_version(
|
| 614 |
+
registry, collection, aliases, None, None, bucket_id=bucket_id
|
| 615 |
+
)
|
| 616 |
+
|
| 617 |
+
def unlink(self) -> None:
|
| 618 |
+
"""Remove this registry link.
|
| 619 |
+
|
| 620 |
+
Call this on the linked artifact returned by `link` or
|
| 621 |
+
`Run.link_artifact`, not on a source artifact version. The source
|
| 622 |
+
artifact and its files are untouched; only the collection membership
|
| 623 |
+
is removed. Aliases pointing at the link go with it, and the
|
| 624 |
+
collection version number is never reused. If the version held
|
| 625 |
+
`latest`, that alias moves to the highest remaining version.
|
| 626 |
+
"""
|
| 627 |
+
if not self.is_link:
|
| 628 |
+
raise ValueError(
|
| 629 |
+
"unlink() removes a registry link; call it on the linked "
|
| 630 |
+
"artifact returned by link() or Run.link_artifact, not on a "
|
| 631 |
+
"source artifact version."
|
| 632 |
+
)
|
| 633 |
+
registry_backend(self._registry_bucket_id).unlink(
|
| 634 |
+
self._registry, self._name, self._version
|
| 635 |
+
)
|
| 636 |
+
|
| 637 |
+
def _resolve_link_source(self) -> tuple[str, str, int]:
|
| 638 |
+
"""Coordinates ``(project, artifact, version)`` a link operation
|
| 639 |
+
records for this artifact. A registry-linked artifact links its
|
| 640 |
+
source version directly, so links never chain."""
|
| 641 |
+
if self.is_link:
|
| 642 |
+
return self._source_project, self._source_artifact, self._source_version
|
| 643 |
+
return self._project, self._name, self._version
|
| 644 |
+
|
| 645 |
+
def _link_version(
|
| 646 |
+
self,
|
| 647 |
+
registry: str,
|
| 648 |
+
collection: str,
|
| 649 |
+
aliases: list[str] | None,
|
| 650 |
+
run_name: str | None,
|
| 651 |
+
run_id: str | None,
|
| 652 |
+
bucket_id: str | None = None,
|
| 653 |
+
) -> "Artifact":
|
| 654 |
+
"""Link this logged artifact version into `registry`/`collection` and
|
| 655 |
+
return the linked artifact hydrated at its registry location.
|
| 656 |
+
|
| 657 |
+
A bucket-backed registry accepts a version whose bytes live on a Space
|
| 658 |
+
or a self-hosted server, because the link records where the source is
|
| 659 |
+
and every reader of the bucket can follow it. A local registry cannot:
|
| 660 |
+
it is reachable only from this machine, so linking remote bytes into it
|
| 661 |
+
would produce a pointer nobody else can resolve.
|
| 662 |
+
"""
|
| 663 |
+
source_project, source_artifact, source_version = self._resolve_link_source()
|
| 664 |
+
if self.is_link:
|
| 665 |
+
utils._emit_nonfatal_warning(
|
| 666 |
+
f"Linking a linked artifact links its source version "
|
| 667 |
+
f"({self.source_qualified_name}) directly; registry links "
|
| 668 |
+
"never chain."
|
| 669 |
+
)
|
| 670 |
+
bucket_id = utils.resolve_registry_bucket_id(bucket_id)
|
| 671 |
+
if self._remote_source is not None and bucket_id is None:
|
| 672 |
+
raise NotImplementedError(
|
| 673 |
+
"Linking an artifact fetched from a Space or a self-hosted "
|
| 674 |
+
"server into a local registry is not supported. Pass "
|
| 675 |
+
"bucket_id= (or set TRACKIO_REGISTRY_BUCKET_ID) to publish "
|
| 676 |
+
"into a bucket-backed registry both sides can reach."
|
| 677 |
+
)
|
| 678 |
+
manifest = self._manifest or []
|
| 679 |
+
remote_source = self._remote_source or {}
|
| 680 |
+
link = registry_backend(bucket_id).link_artifact_version(
|
| 681 |
+
registry=registry,
|
| 682 |
+
collection=collection,
|
| 683 |
+
type=self._type,
|
| 684 |
+
source_project=source_project,
|
| 685 |
+
source_artifact=source_artifact,
|
| 686 |
+
source_version=source_version,
|
| 687 |
+
aliases=aliases,
|
| 688 |
+
run_name=run_name,
|
| 689 |
+
run_id=run_id,
|
| 690 |
+
source_space_id=remote_source.get("space_id"),
|
| 691 |
+
source_bucket_id=remote_source.get("bucket_id"),
|
| 692 |
+
source_server_base_url=remote_source.get("server_base_url"),
|
| 693 |
+
)
|
| 694 |
+
linked = Artifact._from_registry_link(
|
| 695 |
+
registry,
|
| 696 |
+
collection,
|
| 697 |
+
{
|
| 698 |
+
**link,
|
| 699 |
+
"source": {
|
| 700 |
+
"type": self._type,
|
| 701 |
+
"description": self._description,
|
| 702 |
+
"metadata": self._metadata,
|
| 703 |
+
"manifest": manifest,
|
| 704 |
+
"manifest_digest": self._manifest_digest,
|
| 705 |
+
"size_bytes": self._size or 0,
|
| 706 |
+
},
|
| 707 |
+
},
|
| 708 |
+
bucket_id=bucket_id,
|
| 709 |
+
)
|
| 710 |
+
if self._remote_source is not None:
|
| 711 |
+
linked._remote_source = dict(self._remote_source)
|
| 712 |
+
return linked
|
| 713 |
+
|
| 714 |
+
@classmethod
|
| 715 |
+
def _from_registry_link(
|
| 716 |
+
cls,
|
| 717 |
+
registry: str,
|
| 718 |
+
collection: str,
|
| 719 |
+
link: dict,
|
| 720 |
+
bucket_id: str | None = None,
|
| 721 |
+
) -> "Artifact":
|
| 722 |
+
"""Build the linked artifact at its registry location from a link
|
| 723 |
+
record. `bucket_id` is remembered so `unlink()` goes back to the same
|
| 724 |
+
registry the link was written to."""
|
| 725 |
+
source = link["source"]
|
| 726 |
+
linked = cls(
|
| 727 |
+
name=collection,
|
| 728 |
+
type=source["type"],
|
| 729 |
+
description=source.get("description"),
|
| 730 |
+
metadata=source.get("metadata"),
|
| 731 |
+
)
|
| 732 |
+
linked._hydrate_from_db(
|
| 733 |
+
project=registry_project_name(registry),
|
| 734 |
+
version=int(link["collection_version"]),
|
| 735 |
+
aliases=link["aliases"],
|
| 736 |
+
manifest=source["manifest"],
|
| 737 |
+
manifest_digest=source["manifest_digest"],
|
| 738 |
+
size_bytes=int(source["size_bytes"]),
|
| 739 |
+
)
|
| 740 |
+
linked._registry = registry
|
| 741 |
+
linked._registry_bucket_id = bucket_id
|
| 742 |
+
linked._source_project = link["source_project"]
|
| 743 |
+
linked._source_artifact = link["source_artifact"]
|
| 744 |
+
linked._source_version = int(link["source_version"])
|
| 745 |
+
return linked
|
| 746 |
+
|
| 747 |
+
@property
|
| 748 |
+
def references(self) -> list[dict]:
|
| 749 |
+
"""Reference entries (each with `path`, `ref`, `size`, and a `digest`)
|
| 750 |
+
recorded via `add_reference`. These point at external data not stored in
|
| 751 |
+
Trackio; read them to work with the URIs without downloading (unlike
|
| 752 |
+
`download()`, which fetches them). Returns a fresh copy on each access;
|
| 753 |
+
empty if the artifact has no references or has not been logged yet."""
|
| 754 |
+
if self._manifest is None:
|
| 755 |
+
return []
|
| 756 |
+
return [dict(e) for e in self._manifest if references.is_reference_entry(e)]
|
| 757 |
+
|
| 758 |
+
def get_entry_uri(self, name: str) -> str | None:
|
| 759 |
+
"""Return the URI recorded for the reference entry at logical path
|
| 760 |
+
`name`, or None if there is no reference entry with that name."""
|
| 761 |
+
if self._manifest is None:
|
| 762 |
+
return None
|
| 763 |
+
for entry in self._manifest:
|
| 764 |
+
if entry.get("path") == name and references.is_reference_entry(entry):
|
| 765 |
+
return entry["ref"]
|
| 766 |
+
return None
|
| 767 |
+
|
| 768 |
+
def __repr__(self) -> str:
|
| 769 |
+
parts: list[str] = [f"name={self._name!r}", f"type={self._type!r}"]
|
| 770 |
+
if self._version is not None:
|
| 771 |
+
parts.append(f"version=v{self._version}")
|
| 772 |
+
if self._aliases:
|
| 773 |
+
parts.append(f"aliases={list(self._aliases)!r}")
|
| 774 |
+
return f"Artifact({', '.join(parts)})"
|
trackio/asgi_app.py
ADDED
|
@@ -0,0 +1,538 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import inspect
|
| 4 |
+
import json
|
| 5 |
+
import logging
|
| 6 |
+
import math
|
| 7 |
+
import secrets
|
| 8 |
+
import tempfile
|
| 9 |
+
import threading
|
| 10 |
+
from collections.abc import Callable
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
from typing import Any, get_args, get_origin
|
| 13 |
+
from urllib.parse import unquote
|
| 14 |
+
|
| 15 |
+
from starlette.applications import Starlette
|
| 16 |
+
from starlette.requests import Request
|
| 17 |
+
from starlette.responses import FileResponse, JSONResponse, Response, StreamingResponse
|
| 18 |
+
from starlette.routing import Route
|
| 19 |
+
|
| 20 |
+
from trackio import utils
|
| 21 |
+
from trackio.exceptions import TrackioAPIError
|
| 22 |
+
from trackio.remote_client import HTTP_API_VERSION
|
| 23 |
+
|
| 24 |
+
logger = logging.getLogger("trackio.asgi_app")
|
| 25 |
+
|
| 26 |
+
_PACKAGE_JSON_PATH = Path(__file__).parent / "package.json"
|
| 27 |
+
_TRACKIO_PACKAGE_VERSION = json.loads(_PACKAGE_JSON_PATH.read_text())["version"]
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _normalize_allowed_file_roots(
|
| 31 |
+
allowed_file_roots: list[str | Path] | None,
|
| 32 |
+
) -> tuple[Path, ...]:
|
| 33 |
+
roots = []
|
| 34 |
+
for root in allowed_file_roots or []:
|
| 35 |
+
roots.append(Path(root).resolve())
|
| 36 |
+
return tuple(roots)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _is_allowed_file_path(path: Path, allowed_roots: tuple[Path, ...]) -> bool:
|
| 40 |
+
resolved_path = path.resolve(strict=False)
|
| 41 |
+
for root in allowed_roots:
|
| 42 |
+
try:
|
| 43 |
+
resolved_path.relative_to(root)
|
| 44 |
+
return True
|
| 45 |
+
except ValueError:
|
| 46 |
+
continue
|
| 47 |
+
return False
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _json_safe(data: Any) -> Any:
|
| 51 |
+
if data is None or isinstance(data, (str, bool, int)):
|
| 52 |
+
return data
|
| 53 |
+
if isinstance(data, float):
|
| 54 |
+
return data if math.isfinite(data) else None
|
| 55 |
+
if isinstance(data, dict):
|
| 56 |
+
return {k: _json_safe(v) for k, v in data.items()}
|
| 57 |
+
if isinstance(data, (list, tuple)):
|
| 58 |
+
return [_json_safe(v) for v in data]
|
| 59 |
+
if hasattr(data, "item"):
|
| 60 |
+
try:
|
| 61 |
+
return _json_safe(data.item())
|
| 62 |
+
except Exception:
|
| 63 |
+
pass
|
| 64 |
+
return str(data)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def register_uploaded_temp_file(request: Request, file_path: str | Path) -> None:
|
| 68 |
+
resolved_path = Path(file_path).resolve(strict=False)
|
| 69 |
+
with request.app.state.uploaded_temp_files_lock:
|
| 70 |
+
request.app.state.uploaded_temp_files.add(resolved_path)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def consume_uploaded_temp_file(request: Request, file_data: Any) -> Path:
|
| 74 |
+
file_path = file_data.get("path") if isinstance(file_data, dict) else None
|
| 75 |
+
if not isinstance(file_path, str) or not file_path:
|
| 76 |
+
raise TrackioAPIError("Expected uploaded file metadata with a valid path.")
|
| 77 |
+
|
| 78 |
+
resolved_path = Path(file_path).resolve(strict=False)
|
| 79 |
+
with request.app.state.uploaded_temp_files_lock:
|
| 80 |
+
if resolved_path not in request.app.state.uploaded_temp_files:
|
| 81 |
+
raise TrackioAPIError(
|
| 82 |
+
"Uploaded file was not created by this Trackio server."
|
| 83 |
+
)
|
| 84 |
+
request.app.state.uploaded_temp_files.remove(resolved_path)
|
| 85 |
+
|
| 86 |
+
if not resolved_path.is_file():
|
| 87 |
+
raise TrackioAPIError("Uploaded file is missing.")
|
| 88 |
+
|
| 89 |
+
return resolved_path
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def cleanup_uploaded_temp_file(file_path: str | Path) -> None:
|
| 93 |
+
try:
|
| 94 |
+
Path(file_path).unlink(missing_ok=True)
|
| 95 |
+
except Exception:
|
| 96 |
+
pass
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _invoke_handler(
|
| 100 |
+
fn: Any,
|
| 101 |
+
request: Request,
|
| 102 |
+
args: list[Any] | None = None,
|
| 103 |
+
kwargs: dict[str, Any] | None = None,
|
| 104 |
+
) -> Any:
|
| 105 |
+
sig = inspect.signature(fn)
|
| 106 |
+
params = list(sig.parameters.values())
|
| 107 |
+
positional_args: list[Any] = []
|
| 108 |
+
keyword_args: dict[str, Any] = {}
|
| 109 |
+
args = list(args or [])
|
| 110 |
+
kwargs = dict(kwargs or {})
|
| 111 |
+
data_index = 0
|
| 112 |
+
|
| 113 |
+
for param in params:
|
| 114 |
+
if param.name == "request":
|
| 115 |
+
keyword_args["request"] = request
|
| 116 |
+
elif param.kind == inspect.Parameter.VAR_POSITIONAL:
|
| 117 |
+
positional_args.extend(args[data_index:])
|
| 118 |
+
data_index = len(args)
|
| 119 |
+
elif param.kind == inspect.Parameter.VAR_KEYWORD:
|
| 120 |
+
keyword_args.update(kwargs)
|
| 121 |
+
kwargs.clear()
|
| 122 |
+
elif param.name in kwargs:
|
| 123 |
+
keyword_args[param.name] = kwargs.pop(param.name)
|
| 124 |
+
elif param.kind in (
|
| 125 |
+
inspect.Parameter.POSITIONAL_ONLY,
|
| 126 |
+
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
| 127 |
+
) and data_index < len(args):
|
| 128 |
+
positional_args.append(args[data_index])
|
| 129 |
+
data_index += 1
|
| 130 |
+
elif param.default is inspect.Signature.empty and param.kind not in (
|
| 131 |
+
inspect.Parameter.VAR_POSITIONAL,
|
| 132 |
+
inspect.Parameter.VAR_KEYWORD,
|
| 133 |
+
):
|
| 134 |
+
raise TrackioAPIError(f"Missing required parameter: {param.name}")
|
| 135 |
+
|
| 136 |
+
return fn(*positional_args, **keyword_args)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
async def version_handler(request: Request) -> Response:
|
| 140 |
+
mcp_enabled = bool(getattr(request.app.state, "mcp_enabled", False))
|
| 141 |
+
return JSONResponse(
|
| 142 |
+
{
|
| 143 |
+
"version": _TRACKIO_PACKAGE_VERSION,
|
| 144 |
+
"api_version": HTTP_API_VERSION,
|
| 145 |
+
"writes_paused": utils.writes_paused(),
|
| 146 |
+
"api_transport": "http",
|
| 147 |
+
"mcp_enabled": mcp_enabled,
|
| 148 |
+
"mcp_path": "/mcp" if mcp_enabled else None,
|
| 149 |
+
}
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def _json_schema_and_python_type(annotation: Any) -> tuple[dict[str, Any], str]:
|
| 154 |
+
if annotation is inspect.Parameter.empty:
|
| 155 |
+
return {"type": "object"}, "Any"
|
| 156 |
+
origin = get_origin(annotation)
|
| 157 |
+
args = get_args(annotation)
|
| 158 |
+
if origin is not None and args:
|
| 159 |
+
non_none = tuple(a for a in args if a is not type(None))
|
| 160 |
+
if len(non_none) == 1 and len(args) > 1:
|
| 161 |
+
return _json_schema_and_python_type(non_none[0])
|
| 162 |
+
if origin is list:
|
| 163 |
+
inner, py_inner = _json_schema_and_python_type(
|
| 164 |
+
args[0] if args else inspect.Parameter.empty
|
| 165 |
+
)
|
| 166 |
+
return {"type": "array", "items": inner}, f"list[{py_inner}]"
|
| 167 |
+
if origin is dict:
|
| 168 |
+
return {"type": "object"}, "dict"
|
| 169 |
+
if annotation in (str, bytes):
|
| 170 |
+
return {"type": "string"}, "str"
|
| 171 |
+
if annotation is int:
|
| 172 |
+
return {"type": "integer"}, "int"
|
| 173 |
+
if annotation is float:
|
| 174 |
+
return {"type": "number"}, "float"
|
| 175 |
+
if annotation is bool:
|
| 176 |
+
return {"type": "boolean"}, "bool"
|
| 177 |
+
name = getattr(annotation, "__name__", None)
|
| 178 |
+
if name:
|
| 179 |
+
return {"type": "object"}, name
|
| 180 |
+
return {"type": "object"}, "Any"
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def build_gradio_api_info(api_registry: dict[str, Any]) -> dict[str, Any]:
|
| 184 |
+
named_endpoints: dict[str, Any] = {}
|
| 185 |
+
for name in sorted(api_registry.keys()):
|
| 186 |
+
fn = api_registry[name]
|
| 187 |
+
if not callable(fn):
|
| 188 |
+
continue
|
| 189 |
+
sig = inspect.signature(fn)
|
| 190 |
+
parameters: list[dict[str, Any]] = []
|
| 191 |
+
for pname, param in sig.parameters.items():
|
| 192 |
+
if pname == "request":
|
| 193 |
+
continue
|
| 194 |
+
jtype, pytype = _json_schema_and_python_type(param.annotation)
|
| 195 |
+
has_default = param.default is not inspect.Parameter.empty
|
| 196 |
+
parameters.append(
|
| 197 |
+
{
|
| 198 |
+
"label": pname,
|
| 199 |
+
"parameter_name": pname,
|
| 200 |
+
"parameter_has_default": has_default,
|
| 201 |
+
"parameter_default": None if not has_default else param.default,
|
| 202 |
+
"type": jtype,
|
| 203 |
+
"python_type": {"type": pytype, "description": ""},
|
| 204 |
+
"component": "Api",
|
| 205 |
+
"example_input": None,
|
| 206 |
+
}
|
| 207 |
+
)
|
| 208 |
+
ret_ann = sig.return_annotation
|
| 209 |
+
if ret_ann is inspect.Signature.empty:
|
| 210 |
+
ret_ann = Any
|
| 211 |
+
rjtype, rpytype = _json_schema_and_python_type(ret_ann)
|
| 212 |
+
returns = [
|
| 213 |
+
{
|
| 214 |
+
"label": "result",
|
| 215 |
+
"type": rjtype,
|
| 216 |
+
"python_type": {"type": rpytype, "description": ""},
|
| 217 |
+
"component": "Api",
|
| 218 |
+
}
|
| 219 |
+
]
|
| 220 |
+
named_endpoints[f"/{name}"] = {
|
| 221 |
+
"parameters": parameters,
|
| 222 |
+
"returns": returns,
|
| 223 |
+
"api_visibility": "public",
|
| 224 |
+
}
|
| 225 |
+
return {"named_endpoints": named_endpoints, "unnamed_endpoints": {}}
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
_MAX_GRADIO_CALL_EVENTS = 256
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def _hf_token_value_is_unset(value: Any) -> bool:
|
| 232 |
+
if value is None:
|
| 233 |
+
return True
|
| 234 |
+
if isinstance(value, str) and value.strip() == "":
|
| 235 |
+
return True
|
| 236 |
+
return False
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def _authorization_bearer_token(request: Request) -> str | None:
|
| 240 |
+
auth = request.headers.get("authorization") or request.headers.get("Authorization")
|
| 241 |
+
if not auth:
|
| 242 |
+
return None
|
| 243 |
+
parts = auth.split()
|
| 244 |
+
if len(parts) != 2 or parts[0].lower() != "bearer":
|
| 245 |
+
return None
|
| 246 |
+
tok = parts[1].strip()
|
| 247 |
+
return tok or None
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def _maybe_apply_hf_token_from_authorization(
|
| 251 |
+
request: Request, fn: Any, args: list[Any], kwargs: dict[str, Any]
|
| 252 |
+
) -> None:
|
| 253 |
+
if not utils.on_spaces():
|
| 254 |
+
return
|
| 255 |
+
token = _authorization_bearer_token(request)
|
| 256 |
+
if not token:
|
| 257 |
+
return
|
| 258 |
+
sig = inspect.signature(fn)
|
| 259 |
+
if "hf_token" not in sig.parameters:
|
| 260 |
+
return
|
| 261 |
+
params = [p for p in sig.parameters.values() if p.name != "request"]
|
| 262 |
+
names = [p.name for p in params]
|
| 263 |
+
if "hf_token" not in names:
|
| 264 |
+
return
|
| 265 |
+
idx = names.index("hf_token")
|
| 266 |
+
if "hf_token" in kwargs:
|
| 267 |
+
if _hf_token_value_is_unset(kwargs["hf_token"]):
|
| 268 |
+
kwargs["hf_token"] = token
|
| 269 |
+
return
|
| 270 |
+
if idx < len(args):
|
| 271 |
+
if _hf_token_value_is_unset(args[idx]):
|
| 272 |
+
args[idx] = token
|
| 273 |
+
return
|
| 274 |
+
kwargs["hf_token"] = token
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def _store_gradio_call_result(
|
| 278 |
+
request: Request, event_id: str, api_name: str, data: Any
|
| 279 |
+
) -> None:
|
| 280 |
+
with request.app.state.gradio_call_events_lock:
|
| 281 |
+
d = request.app.state.gradio_call_events
|
| 282 |
+
while len(d) >= _MAX_GRADIO_CALL_EVENTS:
|
| 283 |
+
d.pop(next(iter(d)))
|
| 284 |
+
d[event_id] = {"api_name": api_name, "data": data}
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
async def run_api_request(request: Request, api_name: str) -> Response:
|
| 288 |
+
api_registry = request.app.state.api_registry
|
| 289 |
+
fn = api_registry.get(api_name)
|
| 290 |
+
if fn is None:
|
| 291 |
+
return JSONResponse({"error": f"Unknown API: {api_name}"}, status_code=404)
|
| 292 |
+
if api_name in request.app.state.write_apis and utils.writes_paused():
|
| 293 |
+
return JSONResponse({"error": utils.WRITES_PAUSED_MESSAGE}, status_code=503)
|
| 294 |
+
|
| 295 |
+
try:
|
| 296 |
+
body = await request.json()
|
| 297 |
+
except Exception:
|
| 298 |
+
body = {}
|
| 299 |
+
|
| 300 |
+
args: list[Any] = []
|
| 301 |
+
kwargs: dict[str, Any] = {}
|
| 302 |
+
if isinstance(body, dict):
|
| 303 |
+
if "args" in body or "kwargs" in body:
|
| 304 |
+
args = body.get("args") or []
|
| 305 |
+
kwargs = body.get("kwargs") or {}
|
| 306 |
+
elif "data" in body and isinstance(body["data"], list):
|
| 307 |
+
args = body["data"]
|
| 308 |
+
else:
|
| 309 |
+
kwargs = body
|
| 310 |
+
elif isinstance(body, list):
|
| 311 |
+
args = body
|
| 312 |
+
elif body is not None:
|
| 313 |
+
args = [body]
|
| 314 |
+
|
| 315 |
+
if not isinstance(args, list):
|
| 316 |
+
args = [args]
|
| 317 |
+
if not isinstance(kwargs, dict):
|
| 318 |
+
kwargs = {}
|
| 319 |
+
|
| 320 |
+
_maybe_apply_hf_token_from_authorization(request, fn, args, kwargs)
|
| 321 |
+
|
| 322 |
+
try:
|
| 323 |
+
result = _invoke_handler(fn, request, args=args, kwargs=kwargs)
|
| 324 |
+
if inspect.isawaitable(result):
|
| 325 |
+
result = await result
|
| 326 |
+
return JSONResponse({"data": _json_safe(result)})
|
| 327 |
+
except TrackioAPIError as e:
|
| 328 |
+
return JSONResponse({"error": str(e)}, status_code=400)
|
| 329 |
+
except Exception as e:
|
| 330 |
+
return JSONResponse({"error": str(e)}, status_code=500)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
async def api_handler(request: Request) -> Response:
|
| 334 |
+
return await run_api_request(request, request.path_params["api_name"])
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
async def gradio_api_info_handler(request: Request) -> Response:
|
| 338 |
+
return JSONResponse(
|
| 339 |
+
build_gradio_api_info(request.app.state.api_registry),
|
| 340 |
+
headers={"Cache-Control": "no-store"},
|
| 341 |
+
)
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
async def gradio_call_post_handler(request: Request) -> Response:
|
| 345 |
+
api_name = request.path_params["api_name"]
|
| 346 |
+
resp = await run_api_request(request, api_name)
|
| 347 |
+
if resp.status_code != 200:
|
| 348 |
+
return resp
|
| 349 |
+
body = json.loads(bytes(resp.body).decode())
|
| 350 |
+
event_id = secrets.token_urlsafe(16)
|
| 351 |
+
_store_gradio_call_result(request, event_id, api_name, body["data"])
|
| 352 |
+
return JSONResponse({"event_id": event_id})
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
async def gradio_call_poll_handler(request: Request) -> Response:
|
| 356 |
+
api_name = request.path_params["api_name"]
|
| 357 |
+
event_id = request.path_params["event_id"]
|
| 358 |
+
with request.app.state.gradio_call_events_lock:
|
| 359 |
+
event = request.app.state.gradio_call_events.pop(event_id, None)
|
| 360 |
+
if event is None:
|
| 361 |
+
logger.info("gradio_api poll: unknown or expired event_id")
|
| 362 |
+
return JSONResponse({"error": "Unknown or expired event_id"}, status_code=404)
|
| 363 |
+
if event.get("api_name") != api_name:
|
| 364 |
+
logger.info(
|
| 365 |
+
"gradio_api poll: api_name mismatch (path=%r, stored=%r)",
|
| 366 |
+
api_name,
|
| 367 |
+
event.get("api_name"),
|
| 368 |
+
)
|
| 369 |
+
with request.app.state.gradio_call_events_lock:
|
| 370 |
+
d = request.app.state.gradio_call_events
|
| 371 |
+
while len(d) >= _MAX_GRADIO_CALL_EVENTS:
|
| 372 |
+
d.pop(next(iter(d)))
|
| 373 |
+
d[event_id] = event
|
| 374 |
+
return JSONResponse({"error": "Unknown or expired event_id"}, status_code=404)
|
| 375 |
+
|
| 376 |
+
data = event["data"]
|
| 377 |
+
payload = json.dumps(_json_safe([data]))
|
| 378 |
+
|
| 379 |
+
async def sse() -> Any:
|
| 380 |
+
yield f"event: complete\ndata: {payload}\n\n"
|
| 381 |
+
|
| 382 |
+
return StreamingResponse(
|
| 383 |
+
sse(),
|
| 384 |
+
media_type="text/event-stream",
|
| 385 |
+
headers={
|
| 386 |
+
"Cache-Control": "no-store",
|
| 387 |
+
"X-Accel-Buffering": "no",
|
| 388 |
+
},
|
| 389 |
+
)
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
async def upload_handler(request: Request) -> Response:
|
| 393 |
+
upload_authorizer = getattr(request.app.state, "upload_authorizer", None)
|
| 394 |
+
if callable(upload_authorizer):
|
| 395 |
+
try:
|
| 396 |
+
upload_authorizer(request)
|
| 397 |
+
except TrackioAPIError as e:
|
| 398 |
+
return JSONResponse({"error": str(e)}, status_code=400)
|
| 399 |
+
|
| 400 |
+
form = await request.form()
|
| 401 |
+
uploads = form.getlist("files")
|
| 402 |
+
saved_paths = []
|
| 403 |
+
for upload in uploads:
|
| 404 |
+
suffix = Path(getattr(upload, "filename", "") or "").suffix
|
| 405 |
+
with tempfile.NamedTemporaryFile(
|
| 406 |
+
delete=False,
|
| 407 |
+
prefix="trackio-upload-",
|
| 408 |
+
suffix=suffix,
|
| 409 |
+
) as tmp:
|
| 410 |
+
tmp.write(await upload.read())
|
| 411 |
+
register_uploaded_temp_file(request, tmp.name)
|
| 412 |
+
saved_paths.append(tmp.name)
|
| 413 |
+
return JSONResponse({"paths": saved_paths})
|
| 414 |
+
|
| 415 |
+
|
| 416 |
+
async def gradio_upload_alias_handler(request: Request) -> Response:
|
| 417 |
+
return await upload_handler(request)
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
_DISALLOWED_FILE_SUFFIXES = frozenset(
|
| 421 |
+
{".db", ".db-journal", ".db-wal", ".db-shm", ".sqlite", ".sqlite3"}
|
| 422 |
+
)
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
async def file_handler(request: Request) -> Response:
|
| 426 |
+
fs_path = request.query_params.get("path")
|
| 427 |
+
if fs_path is None:
|
| 428 |
+
return Response("Missing path", status_code=400)
|
| 429 |
+
fp = Path(unquote(fs_path)).resolve(strict=False)
|
| 430 |
+
if fp.suffix.lower() in _DISALLOWED_FILE_SUFFIXES:
|
| 431 |
+
return Response("Not found", status_code=404)
|
| 432 |
+
if _is_allowed_file_path(fp, (utils.ARTIFACTS_DIR.resolve(),)):
|
| 433 |
+
return Response("Not found", status_code=404)
|
| 434 |
+
allowed_roots = getattr(request.app.state, "allowed_file_roots", ())
|
| 435 |
+
if fp.is_file() and _is_allowed_file_path(fp, allowed_roots):
|
| 436 |
+
return FileResponse(str(fp))
|
| 437 |
+
return Response("Not found", status_code=404)
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
async def artifact_blob_handler(request: Request) -> Response:
|
| 441 |
+
from trackio import cas # noqa: PLC0415
|
| 442 |
+
from trackio import server as _server # noqa: PLC0415
|
| 443 |
+
|
| 444 |
+
project = request.path_params.get("project")
|
| 445 |
+
digest = request.path_params.get("digest")
|
| 446 |
+
try:
|
| 447 |
+
project = _server._validate_project_name(project)
|
| 448 |
+
digest = _server._validate_sha256_digest(digest)
|
| 449 |
+
except TrackioAPIError:
|
| 450 |
+
return Response("Not found", status_code=404)
|
| 451 |
+
blob = cas.blob_path(project, digest)
|
| 452 |
+
if not blob.is_file():
|
| 453 |
+
return Response("Not found", status_code=404)
|
| 454 |
+
return FileResponse(str(blob))
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def create_trackio_starlette_app(
|
| 458 |
+
oauth_routes: list[Route],
|
| 459 |
+
api_registry: dict[str, Any],
|
| 460 |
+
extra_routes: list[Any] | None = None,
|
| 461 |
+
write_apis: frozenset[str] = frozenset(),
|
| 462 |
+
mcp_lifespan: Any = None,
|
| 463 |
+
mcp_enabled: bool = False,
|
| 464 |
+
allowed_file_roots: list[str | Path] | None = None,
|
| 465 |
+
upload_authorizer: Callable[[Request], None] | None = None,
|
| 466 |
+
) -> Starlette:
|
| 467 |
+
routes: list[Any] = list(oauth_routes)
|
| 468 |
+
routes.extend(
|
| 469 |
+
[
|
| 470 |
+
Route("/version", endpoint=version_handler, methods=["GET"]),
|
| 471 |
+
Route("/api/upload", endpoint=upload_handler, methods=["POST"]),
|
| 472 |
+
Route("/api/{api_name:str}", endpoint=api_handler, methods=["POST"]),
|
| 473 |
+
Route("/file", endpoint=file_handler, methods=["GET"]),
|
| 474 |
+
Route(
|
| 475 |
+
"/artifact_blob/{project:str}/{digest:str}",
|
| 476 |
+
endpoint=artifact_blob_handler,
|
| 477 |
+
methods=["GET"],
|
| 478 |
+
),
|
| 479 |
+
]
|
| 480 |
+
)
|
| 481 |
+
if utils.on_spaces():
|
| 482 |
+
routes.extend(
|
| 483 |
+
[
|
| 484 |
+
Route(
|
| 485 |
+
"/gradio_api/info",
|
| 486 |
+
endpoint=gradio_api_info_handler,
|
| 487 |
+
methods=["GET"],
|
| 488 |
+
),
|
| 489 |
+
Route(
|
| 490 |
+
"/gradio_api/info/",
|
| 491 |
+
endpoint=gradio_api_info_handler,
|
| 492 |
+
methods=["GET"],
|
| 493 |
+
),
|
| 494 |
+
Route(
|
| 495 |
+
"/gradio_api/upload",
|
| 496 |
+
endpoint=gradio_upload_alias_handler,
|
| 497 |
+
methods=["POST"],
|
| 498 |
+
),
|
| 499 |
+
Route(
|
| 500 |
+
"/gradio_api/upload/",
|
| 501 |
+
endpoint=gradio_upload_alias_handler,
|
| 502 |
+
methods=["POST"],
|
| 503 |
+
),
|
| 504 |
+
Route(
|
| 505 |
+
"/gradio_api/call/{api_name:str}",
|
| 506 |
+
endpoint=gradio_call_post_handler,
|
| 507 |
+
methods=["POST"],
|
| 508 |
+
),
|
| 509 |
+
Route(
|
| 510 |
+
"/gradio_api/call/{api_name:str}/",
|
| 511 |
+
endpoint=gradio_call_post_handler,
|
| 512 |
+
methods=["POST"],
|
| 513 |
+
),
|
| 514 |
+
Route(
|
| 515 |
+
"/gradio_api/call/{api_name:str}/{event_id:str}",
|
| 516 |
+
endpoint=gradio_call_poll_handler,
|
| 517 |
+
methods=["GET"],
|
| 518 |
+
),
|
| 519 |
+
Route(
|
| 520 |
+
"/gradio_api/call/{api_name:str}/{event_id:str}/",
|
| 521 |
+
endpoint=gradio_call_poll_handler,
|
| 522 |
+
methods=["GET"],
|
| 523 |
+
),
|
| 524 |
+
]
|
| 525 |
+
)
|
| 526 |
+
routes.extend(extra_routes or [])
|
| 527 |
+
app = Starlette(routes=routes, lifespan=mcp_lifespan)
|
| 528 |
+
app.state.api_registry = api_registry
|
| 529 |
+
app.state.write_apis = write_apis
|
| 530 |
+
app.state.mcp_enabled = mcp_enabled
|
| 531 |
+
app.state.allowed_file_roots = _normalize_allowed_file_roots(allowed_file_roots)
|
| 532 |
+
app.state.upload_authorizer = upload_authorizer
|
| 533 |
+
app.state.uploaded_temp_files = set()
|
| 534 |
+
app.state.uploaded_temp_files_lock = threading.Lock()
|
| 535 |
+
if utils.on_spaces():
|
| 536 |
+
app.state.gradio_call_events = {}
|
| 537 |
+
app.state.gradio_call_events_lock = threading.Lock()
|
| 538 |
+
return app
|
trackio/assets/badge.png
ADDED
|
Git LFS Details
|
trackio/assets/trackio_logo_dark.png
ADDED
|
trackio/assets/trackio_logo_light.png
ADDED
|
trackio/assets/trackio_logo_old.png
ADDED
|
Git LFS Details
|
trackio/assets/trackio_logo_type_dark.png
ADDED
|
trackio/assets/trackio_logo_type_dark_transparent.png
ADDED
|
trackio/assets/trackio_logo_type_light.png
ADDED
|
trackio/assets/trackio_logo_type_light_transparent.png
ADDED
|
trackio/bucket_storage.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import tempfile
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
import huggingface_hub
|
| 5 |
+
|
| 6 |
+
from trackio.cas import is_partial_blob
|
| 7 |
+
from trackio.sqlite_storage import SQLiteStorage
|
| 8 |
+
from trackio.utils import (
|
| 9 |
+
TRACKIO_DIR,
|
| 10 |
+
canonical_project_name,
|
| 11 |
+
project_artifacts_dir,
|
| 12 |
+
project_media_dir,
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def create_bucket_if_not_exists(bucket_id: str, private: bool | None = None) -> None:
|
| 17 |
+
huggingface_hub.create_bucket(bucket_id, private=private, exist_ok=True)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
LOGBOOK_FILES_BUCKET_PREFIX = "logbook-files"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def upload_logbook_path_artifacts_to_bucket(
|
| 24 |
+
bucket_id: str, entries: list[dict]
|
| 25 |
+
) -> list[str]:
|
| 26 |
+
"""Uploads local files referenced by path-artifact cells, preserving their
|
| 27 |
+
relative path under a stable bucket prefix. Returns the relative paths that
|
| 28 |
+
were actually uploaded (missing files are skipped)."""
|
| 29 |
+
additions: list[tuple[str, str]] = []
|
| 30 |
+
uploaded: list[str] = []
|
| 31 |
+
for entry in entries:
|
| 32 |
+
local_path = Path(entry["abs_path"])
|
| 33 |
+
if not local_path.is_file():
|
| 34 |
+
continue
|
| 35 |
+
remote_path = f"{LOGBOOK_FILES_BUCKET_PREFIX}/{entry['path']}"
|
| 36 |
+
additions.append((str(local_path), remote_path))
|
| 37 |
+
uploaded.append(entry["path"])
|
| 38 |
+
if additions:
|
| 39 |
+
huggingface_hub.batch_bucket_files(
|
| 40 |
+
bucket_id, add=additions, token=huggingface_hub.utils.get_token()
|
| 41 |
+
)
|
| 42 |
+
return uploaded
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _list_bucket_file_paths(
|
| 46 |
+
bucket_id: str,
|
| 47 |
+
prefix: str | None = None,
|
| 48 |
+
token: str | bool | None = None,
|
| 49 |
+
) -> list[str]:
|
| 50 |
+
items = huggingface_hub.list_bucket_tree(
|
| 51 |
+
bucket_id, prefix=prefix, recursive=True, token=token
|
| 52 |
+
)
|
| 53 |
+
return [
|
| 54 |
+
item.path
|
| 55 |
+
for item in items
|
| 56 |
+
if getattr(item, "type", None) == "file" and getattr(item, "path", None)
|
| 57 |
+
]
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def download_bucket_to_trackio_dir(bucket_id: str) -> None:
|
| 61 |
+
TRACKIO_DIR.mkdir(parents=True, exist_ok=True)
|
| 62 |
+
huggingface_hub.sync_bucket(
|
| 63 |
+
source=f"hf://buckets/{bucket_id}/trackio",
|
| 64 |
+
dest=str(TRACKIO_DIR),
|
| 65 |
+
quiet=True,
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def upload_project_to_bucket(project: str, bucket_id: str) -> None:
|
| 70 |
+
db_path = SQLiteStorage.get_project_db_path(project)
|
| 71 |
+
if not db_path.exists():
|
| 72 |
+
raise FileNotFoundError(f"No database found for project '{project}'")
|
| 73 |
+
|
| 74 |
+
with SQLiteStorage._get_connection(
|
| 75 |
+
db_path, configure_pragmas=False, row_factory=None
|
| 76 |
+
) as conn:
|
| 77 |
+
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
| 78 |
+
|
| 79 |
+
files_to_add = [(str(db_path), f"trackio/{db_path.name}")]
|
| 80 |
+
|
| 81 |
+
media_dir = project_media_dir(project)
|
| 82 |
+
if media_dir.exists():
|
| 83 |
+
for media_file in media_dir.rglob("*"):
|
| 84 |
+
if media_file.is_file():
|
| 85 |
+
rel = media_file.relative_to(TRACKIO_DIR)
|
| 86 |
+
files_to_add.append((str(media_file), f"trackio/{rel.as_posix()}"))
|
| 87 |
+
|
| 88 |
+
artifacts_dir = project_artifacts_dir(project)
|
| 89 |
+
if artifacts_dir.exists():
|
| 90 |
+
for blob_file in artifacts_dir.rglob("*"):
|
| 91 |
+
if not blob_file.is_file():
|
| 92 |
+
continue
|
| 93 |
+
if is_partial_blob(blob_file.name):
|
| 94 |
+
continue
|
| 95 |
+
rel = blob_file.relative_to(TRACKIO_DIR)
|
| 96 |
+
files_to_add.append((str(blob_file), f"trackio/{rel.as_posix()}"))
|
| 97 |
+
|
| 98 |
+
huggingface_hub.batch_bucket_files(bucket_id, add=files_to_add)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _download_db_from_bucket(
|
| 102 |
+
project: str, bucket_id: str, dest_path: Path | None = None
|
| 103 |
+
) -> bool:
|
| 104 |
+
db_filename = SQLiteStorage.get_project_db_filename(project)
|
| 105 |
+
remote_path = f"trackio/{db_filename}"
|
| 106 |
+
local_path = dest_path or SQLiteStorage.get_project_db_path(project)
|
| 107 |
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
| 108 |
+
try:
|
| 109 |
+
huggingface_hub.download_bucket_files(
|
| 110 |
+
bucket_id,
|
| 111 |
+
files=[(remote_path, str(local_path))],
|
| 112 |
+
token=huggingface_hub.utils.get_token(),
|
| 113 |
+
)
|
| 114 |
+
return local_path.exists()
|
| 115 |
+
except Exception:
|
| 116 |
+
return False
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _local_db_has_data(project: str) -> bool:
|
| 120 |
+
db_path = SQLiteStorage.get_project_db_path(project)
|
| 121 |
+
if not db_path.exists() or db_path.stat().st_size == 0:
|
| 122 |
+
return False
|
| 123 |
+
try:
|
| 124 |
+
with SQLiteStorage._get_connection(
|
| 125 |
+
db_path, configure_pragmas=False, row_factory=None
|
| 126 |
+
) as conn:
|
| 127 |
+
count = conn.execute("SELECT COUNT(*) FROM metrics").fetchone()[0]
|
| 128 |
+
return count > 0
|
| 129 |
+
except Exception:
|
| 130 |
+
return False
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _export_and_upload_static(
|
| 134 |
+
project: str,
|
| 135 |
+
dest_bucket_id: str,
|
| 136 |
+
db_path: Path,
|
| 137 |
+
media_dir: Path | None = None,
|
| 138 |
+
artifacts_dir: Path | None = None,
|
| 139 |
+
) -> None:
|
| 140 |
+
with tempfile.TemporaryDirectory() as tmp_dir:
|
| 141 |
+
output_dir = Path(tmp_dir)
|
| 142 |
+
SQLiteStorage.export_for_static_space(
|
| 143 |
+
project, output_dir, db_path_override=db_path
|
| 144 |
+
)
|
| 145 |
+
|
| 146 |
+
files_to_add = []
|
| 147 |
+
for f in output_dir.rglob("*"):
|
| 148 |
+
if f.is_file():
|
| 149 |
+
rel = f.relative_to(output_dir)
|
| 150 |
+
files_to_add.append((str(f), rel.as_posix()))
|
| 151 |
+
|
| 152 |
+
if media_dir and media_dir.exists():
|
| 153 |
+
for f in media_dir.rglob("*"):
|
| 154 |
+
if f.is_file():
|
| 155 |
+
rel = f.relative_to(media_dir)
|
| 156 |
+
files_to_add.append((str(f), f"media/{rel.as_posix()}"))
|
| 157 |
+
|
| 158 |
+
if artifacts_dir and artifacts_dir.exists():
|
| 159 |
+
for f in artifacts_dir.rglob("*"):
|
| 160 |
+
if f.is_file() and not is_partial_blob(f.name):
|
| 161 |
+
rel = f.relative_to(artifacts_dir)
|
| 162 |
+
files_to_add.append((str(f), f"artifacts/{rel.as_posix()}"))
|
| 163 |
+
|
| 164 |
+
huggingface_hub.batch_bucket_files(dest_bucket_id, add=files_to_add)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def _copy_project_prefix_between_buckets(
|
| 168 |
+
source_bucket_id: str, dest_bucket_id: str, project: str, kind: str
|
| 169 |
+
) -> None:
|
| 170 |
+
source_prefix = f"trackio/{kind}/{canonical_project_name(project)}/"
|
| 171 |
+
files_to_copy = _list_bucket_file_paths(source_bucket_id, prefix=source_prefix)
|
| 172 |
+
if not files_to_copy:
|
| 173 |
+
return
|
| 174 |
+
|
| 175 |
+
huggingface_hub.copy_files(
|
| 176 |
+
f"hf://buckets/{source_bucket_id}/{source_prefix}",
|
| 177 |
+
f"hf://buckets/{dest_bucket_id}/{kind}/",
|
| 178 |
+
)
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def upload_project_to_bucket_for_static(project: str, bucket_id: str) -> None:
|
| 182 |
+
if not _local_db_has_data(project):
|
| 183 |
+
_download_db_from_bucket(project, bucket_id)
|
| 184 |
+
|
| 185 |
+
db_path = SQLiteStorage.get_project_db_path(project)
|
| 186 |
+
_export_and_upload_static(
|
| 187 |
+
project,
|
| 188 |
+
bucket_id,
|
| 189 |
+
db_path,
|
| 190 |
+
project_media_dir(project),
|
| 191 |
+
project_artifacts_dir(project),
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def export_from_bucket_for_static(
|
| 196 |
+
source_bucket_id: str,
|
| 197 |
+
dest_bucket_id: str,
|
| 198 |
+
project: str,
|
| 199 |
+
) -> None:
|
| 200 |
+
with tempfile.TemporaryDirectory() as work_dir:
|
| 201 |
+
work_path = Path(work_dir)
|
| 202 |
+
db_path = work_path / SQLiteStorage.get_project_db_filename(project)
|
| 203 |
+
|
| 204 |
+
if not _download_db_from_bucket(project, source_bucket_id, dest_path=db_path):
|
| 205 |
+
raise FileNotFoundError(
|
| 206 |
+
f"Could not download database for project '{project}' "
|
| 207 |
+
f"from bucket '{source_bucket_id}'."
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
_export_and_upload_static(project, dest_bucket_id, db_path)
|
| 211 |
+
for kind in ("media", "artifacts"):
|
| 212 |
+
_copy_project_prefix_between_buckets(
|
| 213 |
+
source_bucket_id, dest_bucket_id, project, kind
|
| 214 |
+
)
|
trackio/bump.py
ADDED
|
@@ -0,0 +1,614 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Upgrade an existing Trackio Space to the locally installed Trackio version.
|
| 2 |
+
|
| 3 |
+
`bump()` either upgrades a Space in place or duplicates it (and its bucket) into
|
| 4 |
+
a new Space running the local version. The Space's SQLite databases live in the
|
| 5 |
+
Hugging Face Bucket mounted at `/data`; the new Trackio server migrates them when
|
| 6 |
+
it opens them on boot. While an in-place bump is in progress the Space is paused,
|
| 7 |
+
so training clients write their logs to the bucket inbox as JSONL fragments,
|
| 8 |
+
which the upgraded Space imports once it is running again.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import io
|
| 12 |
+
import json
|
| 13 |
+
import re
|
| 14 |
+
import sqlite3
|
| 15 |
+
import tempfile
|
| 16 |
+
import time
|
| 17 |
+
from datetime import datetime, timezone
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import httpx
|
| 21 |
+
import huggingface_hub
|
| 22 |
+
from huggingface_hub.errors import (
|
| 23 |
+
BucketNotFoundError,
|
| 24 |
+
HfHubHTTPError,
|
| 25 |
+
RepositoryNotFoundError,
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
import trackio
|
| 29 |
+
from trackio import deploy
|
| 30 |
+
from trackio.bucket_storage import _list_bucket_file_paths, create_bucket_if_not_exists
|
| 31 |
+
from trackio.remote_client import RemoteClient, _space_id_to_url
|
| 32 |
+
from trackio.sqlite_storage import DB_EXT, SCHEMA_VERSION, SQLiteStorage
|
| 33 |
+
from trackio.utils import PAUSE_WRITES_VARIABLE, preprocess_space_and_dataset_ids
|
| 34 |
+
|
| 35 |
+
MIN_BUMPABLE_VERSION = "0.21.0"
|
| 36 |
+
BACKUP_PREFIX = "trackio-backups"
|
| 37 |
+
DB_PREFIX = "trackio/"
|
| 38 |
+
_JOURNAL_SUFFIX = "-journal"
|
| 39 |
+
_REQUIREMENT_PATTERN = re.compile(
|
| 40 |
+
r"^\s*trackio(?:\[[^\]]*\])?\s*==\s*([0-9][0-9A-Za-z.+\-]*)\s*$"
|
| 41 |
+
)
|
| 42 |
+
_VERSION_PATTERN = re.compile(r"^(\d+)\.(\d+)\.(\d+)$")
|
| 43 |
+
_MANAGED_VARIABLES = frozenset({"TRACKIO_DIR", "TRACKIO_BUCKET_ID"})
|
| 44 |
+
_RUN_COUNT_PAGE_SIZE = 5000
|
| 45 |
+
_FAILURE_STAGES = frozenset(
|
| 46 |
+
("NO_APP_FILE", "CONFIG_ERROR", "BUILD_ERROR", "RUNTIME_ERROR")
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class BumpError(RuntimeError):
|
| 51 |
+
pass
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _version_tuple(version: str) -> tuple[int, int, int] | None:
|
| 55 |
+
match = _VERSION_PATTERN.match(version)
|
| 56 |
+
if match is None:
|
| 57 |
+
return None
|
| 58 |
+
return tuple(int(part) for part in match.groups())
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def get_space_trackio_version(
|
| 62 |
+
space_id: str, hf_api: huggingface_hub.HfApi | None = None
|
| 63 |
+
) -> str | None:
|
| 64 |
+
"""
|
| 65 |
+
Returns the Trackio version a Space is configured to run, read from its
|
| 66 |
+
`requirements.txt` pin, or from `trackio/package.json` for Spaces deployed from
|
| 67 |
+
a source install. Returns `None` when the version cannot be determined.
|
| 68 |
+
"""
|
| 69 |
+
hf_api = hf_api or huggingface_hub.HfApi()
|
| 70 |
+
try:
|
| 71 |
+
path = hf_api.hf_hub_download(space_id, "requirements.txt", repo_type="space")
|
| 72 |
+
for line in Path(path).read_text(encoding="utf-8").splitlines():
|
| 73 |
+
if match := _REQUIREMENT_PATTERN.match(line):
|
| 74 |
+
return match.group(1)
|
| 75 |
+
except (HfHubHTTPError, OSError):
|
| 76 |
+
pass
|
| 77 |
+
try:
|
| 78 |
+
path = hf_api.hf_hub_download(
|
| 79 |
+
space_id, "trackio/package.json", repo_type="space"
|
| 80 |
+
)
|
| 81 |
+
return json.loads(Path(path).read_text(encoding="utf-8")).get("version")
|
| 82 |
+
except (HfHubHTTPError, OSError, ValueError):
|
| 83 |
+
return None
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _check_bumpable(space_id: str, space_version: str | None) -> str:
|
| 87 |
+
local = _version_tuple(trackio.__version__)
|
| 88 |
+
if local is None:
|
| 89 |
+
raise BumpError(
|
| 90 |
+
f"The local Trackio version {trackio.__version__!r} is not a final "
|
| 91 |
+
"release, so it cannot be compared safely with the Space's version."
|
| 92 |
+
)
|
| 93 |
+
if space_version is None:
|
| 94 |
+
raise BumpError(
|
| 95 |
+
f"Could not determine which Trackio version Space '{space_id}' runs. "
|
| 96 |
+
"`trackio bump` only supports Spaces deployed by Trackio."
|
| 97 |
+
)
|
| 98 |
+
remote = _version_tuple(space_version)
|
| 99 |
+
if remote is None:
|
| 100 |
+
raise BumpError(
|
| 101 |
+
f"Space '{space_id}' pins Trackio {space_version!r}, which is not a final "
|
| 102 |
+
"release that `trackio bump` can compare safely."
|
| 103 |
+
)
|
| 104 |
+
if remote < _version_tuple(MIN_BUMPABLE_VERSION):
|
| 105 |
+
raise BumpError(
|
| 106 |
+
f"Space '{space_id}' runs Trackio {space_version}, which is older than "
|
| 107 |
+
f"{MIN_BUMPABLE_VERSION}, the oldest version `trackio bump` supports."
|
| 108 |
+
)
|
| 109 |
+
if remote > local:
|
| 110 |
+
raise BumpError(
|
| 111 |
+
f"Space '{space_id}' runs Trackio {space_version}, which is newer than the "
|
| 112 |
+
f"local Trackio {trackio.__version__}. Upgrade Trackio locally instead of downgrading the Space."
|
| 113 |
+
)
|
| 114 |
+
return space_version
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _bucket_db_paths(bucket_id: str, prefix: str = DB_PREFIX) -> list[str]:
|
| 118 |
+
return sorted(
|
| 119 |
+
path
|
| 120 |
+
for path in _list_bucket_file_paths(bucket_id, prefix=prefix)
|
| 121 |
+
if "/" not in path[len(prefix) :]
|
| 122 |
+
and (path.endswith(DB_EXT) or path.endswith(DB_EXT + _JOURNAL_SUFFIX))
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _db_inventory(db_path: Path) -> dict:
|
| 127 |
+
conn = sqlite3.connect(f"file:{db_path}?mode=ro&immutable=1", uri=True)
|
| 128 |
+
try:
|
| 129 |
+
tables = [
|
| 130 |
+
row[0]
|
| 131 |
+
for row in conn.execute(
|
| 132 |
+
"SELECT name FROM sqlite_master "
|
| 133 |
+
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"
|
| 134 |
+
)
|
| 135 |
+
]
|
| 136 |
+
counts = {
|
| 137 |
+
table: conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]
|
| 138 |
+
for table in tables
|
| 139 |
+
}
|
| 140 |
+
runs = {}
|
| 141 |
+
if "metrics" in tables:
|
| 142 |
+
runs = dict(
|
| 143 |
+
conn.execute("SELECT run_name, COUNT(*) FROM metrics GROUP BY run_name")
|
| 144 |
+
)
|
| 145 |
+
return {"tables": counts, "metric_rows_per_run": runs}
|
| 146 |
+
finally:
|
| 147 |
+
conn.close()
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def bucket_inventory(bucket_id: str, prefix: str = DB_PREFIX) -> dict[str, dict]:
|
| 151 |
+
"""
|
| 152 |
+
Downloads every project database under `prefix` in a bucket and returns, per
|
| 153 |
+
database filename, the row count of each table and the metric rows per run.
|
| 154 |
+
"""
|
| 155 |
+
db_paths = [p for p in _bucket_db_paths(bucket_id, prefix) if p.endswith(DB_EXT)]
|
| 156 |
+
inventory = {}
|
| 157 |
+
with tempfile.TemporaryDirectory() as work_dir:
|
| 158 |
+
for remote_path in db_paths:
|
| 159 |
+
filename = remote_path.rsplit("/", 1)[-1]
|
| 160 |
+
local_path = Path(work_dir) / filename
|
| 161 |
+
huggingface_hub.download_bucket_files(
|
| 162 |
+
bucket_id,
|
| 163 |
+
files=[(remote_path, str(local_path))],
|
| 164 |
+
token=huggingface_hub.utils.get_token(),
|
| 165 |
+
)
|
| 166 |
+
inventory[filename] = _db_inventory(local_path)
|
| 167 |
+
return inventory
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _served_version_info(space_id: str) -> dict:
|
| 171 |
+
headers = {}
|
| 172 |
+
if token := huggingface_hub.utils.get_token():
|
| 173 |
+
headers["Authorization"] = f"Bearer {token}"
|
| 174 |
+
try:
|
| 175 |
+
response = httpx.get(
|
| 176 |
+
_space_id_to_url(space_id) + "version", headers=headers, timeout=30
|
| 177 |
+
)
|
| 178 |
+
response.raise_for_status()
|
| 179 |
+
return response.json()
|
| 180 |
+
except (httpx.HTTPError, ValueError):
|
| 181 |
+
return {}
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def _wait_for_version_info(space_id: str, expected: dict, timeout: int = 300) -> None:
|
| 185 |
+
deadline = time.time() + timeout
|
| 186 |
+
info = {}
|
| 187 |
+
while time.time() < deadline:
|
| 188 |
+
info = _served_version_info(space_id)
|
| 189 |
+
if all(info.get(key) == value for key, value in expected.items()):
|
| 190 |
+
return
|
| 191 |
+
time.sleep(5)
|
| 192 |
+
raise BumpError(
|
| 193 |
+
f"Space '{space_id}' reports {info or 'nothing'}, expected {expected}."
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _metric_rows_per_run(client: RemoteClient, project: str) -> dict[str, int]:
|
| 198 |
+
counts = {}
|
| 199 |
+
offset = 0
|
| 200 |
+
while True:
|
| 201 |
+
rows = client.predict(
|
| 202 |
+
project,
|
| 203 |
+
"SELECT run_name, COUNT(*) AS n FROM metrics GROUP BY run_name "
|
| 204 |
+
f"ORDER BY run_name LIMIT {_RUN_COUNT_PAGE_SIZE} OFFSET {offset}",
|
| 205 |
+
api_name="/query_project",
|
| 206 |
+
)["rows"]
|
| 207 |
+
counts.update((row["run_name"], row["n"]) for row in rows)
|
| 208 |
+
if len(rows) < _RUN_COUNT_PAGE_SIZE:
|
| 209 |
+
return counts
|
| 210 |
+
offset += _RUN_COUNT_PAGE_SIZE
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def verify_space_serves_inventory(space_id: str, inventory: dict[str, dict]) -> None:
|
| 214 |
+
"""
|
| 215 |
+
Checks that a running Space serves the local Trackio version, that every
|
| 216 |
+
project database reached the local `SCHEMA_VERSION`, and that every table
|
| 217 |
+
still holds at least the rows recorded in `inventory`. Rows can only grow,
|
| 218 |
+
since the Space may import inbox fragments.
|
| 219 |
+
"""
|
| 220 |
+
_wait_for_version_info(space_id, {"version": trackio.__version__})
|
| 221 |
+
client = RemoteClient(
|
| 222 |
+
space_id, hf_token=huggingface_hub.utils.get_token(), verbose=False
|
| 223 |
+
)
|
| 224 |
+
projects = client.predict(api_name="/get_all_projects")
|
| 225 |
+
by_filename = {SQLiteStorage.get_project_db_filename(p): p for p in projects}
|
| 226 |
+
for filename, expected in inventory.items():
|
| 227 |
+
project = by_filename.get(filename)
|
| 228 |
+
if project is None:
|
| 229 |
+
raise BumpError(
|
| 230 |
+
f"Space '{space_id}' no longer lists the project stored in {filename}."
|
| 231 |
+
)
|
| 232 |
+
result = client.predict(
|
| 233 |
+
project, "PRAGMA user_version", api_name="/query_project"
|
| 234 |
+
)
|
| 235 |
+
schema_version = result["rows"][0]["user_version"]
|
| 236 |
+
if schema_version != SCHEMA_VERSION:
|
| 237 |
+
raise BumpError(
|
| 238 |
+
f"Project '{project}' is at schema version {schema_version} after the "
|
| 239 |
+
f"bump, expected {SCHEMA_VERSION}."
|
| 240 |
+
)
|
| 241 |
+
for table, count in expected["tables"].items():
|
| 242 |
+
result = client.predict(
|
| 243 |
+
project,
|
| 244 |
+
f'SELECT COUNT(*) AS n FROM "{table}"',
|
| 245 |
+
api_name="/query_project",
|
| 246 |
+
)
|
| 247 |
+
actual = result["rows"][0]["n"]
|
| 248 |
+
if actual < count:
|
| 249 |
+
raise BumpError(
|
| 250 |
+
f"Project '{project}' table '{table}' has {actual} rows after the "
|
| 251 |
+
f"bump, expected at least {count}."
|
| 252 |
+
)
|
| 253 |
+
actual_runs = _metric_rows_per_run(client, project)
|
| 254 |
+
for run, count in expected["metric_rows_per_run"].items():
|
| 255 |
+
if actual_runs.get(run, 0) < count:
|
| 256 |
+
raise BumpError(
|
| 257 |
+
f"Run '{run}' in project '{project}' has {actual_runs.get(run, 0)} "
|
| 258 |
+
f"metric rows after the bump, expected at least {count}."
|
| 259 |
+
)
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def _wait_for_stage(
|
| 263 |
+
space_id: str,
|
| 264 |
+
targets: frozenset[str],
|
| 265 |
+
timeout: int,
|
| 266 |
+
hf_api: huggingface_hub.HfApi,
|
| 267 |
+
) -> None:
|
| 268 |
+
deadline = time.time() + timeout
|
| 269 |
+
stage = None
|
| 270 |
+
while time.time() < deadline:
|
| 271 |
+
try:
|
| 272 |
+
runtime = hf_api.get_space_runtime(space_id)
|
| 273 |
+
stage = str(runtime.stage)
|
| 274 |
+
except (HfHubHTTPError, httpx.RequestError):
|
| 275 |
+
stage = None
|
| 276 |
+
if stage in targets:
|
| 277 |
+
return
|
| 278 |
+
if stage in _FAILURE_STAGES and not targets & _FAILURE_STAGES:
|
| 279 |
+
raise BumpError(
|
| 280 |
+
f"Space '{space_id}' entered {stage}. Check its logs on the Hub."
|
| 281 |
+
)
|
| 282 |
+
time.sleep(5)
|
| 283 |
+
raise BumpError(
|
| 284 |
+
f"Space '{space_id}' did not reach {'/'.join(sorted(targets))} within "
|
| 285 |
+
f"{timeout}s (last stage: {stage})."
|
| 286 |
+
)
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def _runtime_operations(
|
| 290 |
+
space_id: str, hf_api: huggingface_hub.HfApi, revision: str
|
| 291 |
+
) -> list[huggingface_hub.CommitOperationAdd]:
|
| 292 |
+
repo_files = set(
|
| 293 |
+
hf_api.list_repo_files(space_id, repo_type="space", revision=revision)
|
| 294 |
+
)
|
| 295 |
+
has_custom_frontend = any(
|
| 296 |
+
f.startswith(deploy._CUSTOM_SPACE_FRONTEND_DIR + "/") for f in repo_files
|
| 297 |
+
)
|
| 298 |
+
if deploy._is_trackio_installed_from_source():
|
| 299 |
+
requirements = deploy._get_source_install_dependencies()
|
| 300 |
+
else:
|
| 301 |
+
requirements = deploy._get_space_install_requirement()
|
| 302 |
+
app_py = deploy._space_app_py(
|
| 303 |
+
deploy._CUSTOM_SPACE_FRONTEND_DIR if has_custom_frontend else None
|
| 304 |
+
)
|
| 305 |
+
operations = [
|
| 306 |
+
huggingface_hub.CommitOperationAdd(
|
| 307 |
+
"requirements.txt", io.BytesIO(requirements.encode("utf-8"))
|
| 308 |
+
),
|
| 309 |
+
huggingface_hub.CommitOperationAdd(
|
| 310 |
+
"app.py", io.BytesIO(app_py.encode("utf-8"))
|
| 311 |
+
),
|
| 312 |
+
]
|
| 313 |
+
if (
|
| 314 |
+
"trackio/__init__.py" in repo_files
|
| 315 |
+
and not deploy._is_trackio_installed_from_source()
|
| 316 |
+
):
|
| 317 |
+
operations.append(
|
| 318 |
+
huggingface_hub.CommitOperationDelete("trackio/", is_folder=True)
|
| 319 |
+
)
|
| 320 |
+
return operations
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
def _copy_bucket_paths(bucket_id: str, pairs: list[tuple[str, str]]) -> None:
|
| 324 |
+
for source, destination in pairs:
|
| 325 |
+
huggingface_hub.copy_files(
|
| 326 |
+
f"hf://buckets/{bucket_id}/{source}",
|
| 327 |
+
f"hf://buckets/{bucket_id}/{destination}",
|
| 328 |
+
)
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def _bump_in_place(
|
| 332 |
+
space_id: str,
|
| 333 |
+
bucket_id: str,
|
| 334 |
+
space_version: str,
|
| 335 |
+
hf_api: huggingface_hub.HfApi,
|
| 336 |
+
timeout: int,
|
| 337 |
+
) -> str:
|
| 338 |
+
local_version = trackio.__version__
|
| 339 |
+
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
| 340 |
+
backup_prefix = f"{BACKUP_PREFIX}/{space_version}-to-{local_version}-{stamp}/"
|
| 341 |
+
|
| 342 |
+
base_commit = hf_api.repo_info(space_id, repo_type="space").sha
|
| 343 |
+
with tempfile.TemporaryDirectory() as snapshot_dir:
|
| 344 |
+
hf_api.snapshot_download(
|
| 345 |
+
space_id,
|
| 346 |
+
repo_type="space",
|
| 347 |
+
revision=base_commit,
|
| 348 |
+
local_dir=snapshot_dir,
|
| 349 |
+
max_workers=4,
|
| 350 |
+
)
|
| 351 |
+
print(f"* Pausing {space_id}; new logs will queue in the bucket inbox")
|
| 352 |
+
hf_api.pause_space(space_id)
|
| 353 |
+
_wait_for_stage(space_id, frozenset({"PAUSED"}), 300, hf_api)
|
| 354 |
+
|
| 355 |
+
try:
|
| 356 |
+
db_paths = _bucket_db_paths(bucket_id)
|
| 357 |
+
backups = [
|
| 358 |
+
(path, backup_prefix + path[len(DB_PREFIX) :]) for path in db_paths
|
| 359 |
+
]
|
| 360 |
+
_copy_bucket_paths(bucket_id, backups)
|
| 361 |
+
print(
|
| 362 |
+
f"* Backed up {len(db_paths)} database files to "
|
| 363 |
+
f"hf://buckets/{bucket_id}/{backup_prefix}"
|
| 364 |
+
)
|
| 365 |
+
inventory = bucket_inventory(bucket_id, prefix=backup_prefix)
|
| 366 |
+
except Exception as e:
|
| 367 |
+
hf_api.restart_space(space_id)
|
| 368 |
+
raise BumpError(
|
| 369 |
+
f"Backing up '{space_id}' failed before anything was changed, so the "
|
| 370 |
+
f"Space was restarted on Trackio {space_version}: {e}"
|
| 371 |
+
) from e
|
| 372 |
+
|
| 373 |
+
bump_commit = None
|
| 374 |
+
try:
|
| 375 |
+
print(
|
| 376 |
+
f"* Updating {space_id} from Trackio {space_version} to {local_version}"
|
| 377 |
+
)
|
| 378 |
+
bump_commit = hf_api.create_commit(
|
| 379 |
+
space_id,
|
| 380 |
+
operations=_runtime_operations(space_id, hf_api, base_commit),
|
| 381 |
+
commit_message=f"Bump Trackio to {local_version}",
|
| 382 |
+
repo_type="space",
|
| 383 |
+
parent_commit=base_commit,
|
| 384 |
+
).oid
|
| 385 |
+
if deploy._is_trackio_installed_from_source():
|
| 386 |
+
bump_commit = deploy._upload_source_tree(
|
| 387 |
+
hf_api, space_id, parent_commit=bump_commit
|
| 388 |
+
).oid
|
| 389 |
+
hf_api.add_space_variable(space_id, PAUSE_WRITES_VARIABLE, "1")
|
| 390 |
+
hf_api.restart_space(space_id)
|
| 391 |
+
print("* Waiting for the Space to rebuild and migrate its data")
|
| 392 |
+
_wait_for_stage(space_id, frozenset({"RUNNING"}), timeout, hf_api)
|
| 393 |
+
_wait_for_version_info(space_id, {"writes_paused": True})
|
| 394 |
+
verify_space_serves_inventory(space_id, inventory)
|
| 395 |
+
except Exception as e:
|
| 396 |
+
print(f"* Bump failed ({e}); restoring Trackio {space_version}")
|
| 397 |
+
_rollback(
|
| 398 |
+
space_id,
|
| 399 |
+
bucket_id,
|
| 400 |
+
backups,
|
| 401 |
+
Path(snapshot_dir),
|
| 402 |
+
bump_commit,
|
| 403 |
+
hf_api,
|
| 404 |
+
timeout,
|
| 405 |
+
)
|
| 406 |
+
raise BumpError(
|
| 407 |
+
f"Bumping '{space_id}' failed and was rolled back to Trackio "
|
| 408 |
+
f"{space_version}: {e}"
|
| 409 |
+
) from e
|
| 410 |
+
|
| 411 |
+
print("* Verified; resuming writes and importing the queued inbox")
|
| 412 |
+
hf_api.delete_space_variable(space_id, PAUSE_WRITES_VARIABLE)
|
| 413 |
+
try:
|
| 414 |
+
_wait_for_stage(space_id, frozenset({"RUNNING"}), timeout, hf_api)
|
| 415 |
+
_wait_for_version_info(
|
| 416 |
+
space_id, {"version": local_version, "writes_paused": False}, timeout
|
| 417 |
+
)
|
| 418 |
+
except BumpError as e:
|
| 419 |
+
raise BumpError(
|
| 420 |
+
f"{space_id} was upgraded to Trackio {local_version} and verified, but "
|
| 421 |
+
f"did not resume writes: {e} Restart the Space once it is healthy."
|
| 422 |
+
) from e
|
| 423 |
+
print(
|
| 424 |
+
f"* {space_id} now runs Trackio {local_version}: "
|
| 425 |
+
f"{deploy.SPACE_URL.format(space_id=space_id)}"
|
| 426 |
+
)
|
| 427 |
+
return space_id
|
| 428 |
+
|
| 429 |
+
|
| 430 |
+
def _rollback(
|
| 431 |
+
space_id: str,
|
| 432 |
+
bucket_id: str,
|
| 433 |
+
backups: list[tuple[str, str]],
|
| 434 |
+
snapshot_dir: Path,
|
| 435 |
+
bump_commit: str | None,
|
| 436 |
+
hf_api: huggingface_hub.HfApi,
|
| 437 |
+
timeout: int,
|
| 438 |
+
) -> None:
|
| 439 |
+
hf_api.pause_space(space_id)
|
| 440 |
+
_wait_for_stage(space_id, frozenset({"PAUSED"}), 300, hf_api)
|
| 441 |
+
_copy_bucket_paths(bucket_id, [(backup, path) for path, backup in backups])
|
| 442 |
+
try:
|
| 443 |
+
_delete_space_variable(space_id, PAUSE_WRITES_VARIABLE, hf_api)
|
| 444 |
+
if bump_commit is not None:
|
| 445 |
+
hf_api.upload_folder(
|
| 446 |
+
repo_id=space_id,
|
| 447 |
+
repo_type="space",
|
| 448 |
+
folder_path=snapshot_dir,
|
| 449 |
+
ignore_patterns=[".cache/**"],
|
| 450 |
+
delete_patterns=["*", "**/*"],
|
| 451 |
+
commit_message="Roll back failed Trackio bump",
|
| 452 |
+
parent_commit=bump_commit,
|
| 453 |
+
)
|
| 454 |
+
finally:
|
| 455 |
+
hf_api.restart_space(space_id)
|
| 456 |
+
_wait_for_stage(space_id, frozenset({"RUNNING"}) | _FAILURE_STAGES, timeout, hf_api)
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
def _delete_space_variable(
|
| 460 |
+
space_id: str, key: str, hf_api: huggingface_hub.HfApi
|
| 461 |
+
) -> None:
|
| 462 |
+
if key in hf_api.get_space_variables(space_id):
|
| 463 |
+
hf_api.delete_space_variable(space_id, key)
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
def _bump_into_new_space(
|
| 467 |
+
space_id: str,
|
| 468 |
+
bucket_id: str,
|
| 469 |
+
new_space_id: str,
|
| 470 |
+
new_bucket_id: str,
|
| 471 |
+
private: bool,
|
| 472 |
+
hf_api: huggingface_hub.HfApi,
|
| 473 |
+
timeout: int,
|
| 474 |
+
) -> str:
|
| 475 |
+
try:
|
| 476 |
+
hf_api.space_info(new_space_id)
|
| 477 |
+
raise BumpError(f"Space '{new_space_id}' already exists.")
|
| 478 |
+
except RepositoryNotFoundError:
|
| 479 |
+
pass
|
| 480 |
+
try:
|
| 481 |
+
if _list_bucket_file_paths(new_bucket_id):
|
| 482 |
+
raise BumpError(
|
| 483 |
+
f"Bucket '{new_bucket_id}' already exists and is not empty."
|
| 484 |
+
)
|
| 485 |
+
except BucketNotFoundError:
|
| 486 |
+
pass
|
| 487 |
+
|
| 488 |
+
created_bucket = not deploy._bucket_exists(new_bucket_id, hf_api)
|
| 489 |
+
try:
|
| 490 |
+
create_bucket_if_not_exists(
|
| 491 |
+
new_bucket_id, private=bool(hf_api.bucket_info(bucket_id).private)
|
| 492 |
+
)
|
| 493 |
+
if _list_bucket_file_paths(bucket_id):
|
| 494 |
+
print(f"* Copying hf://buckets/{bucket_id} to hf://buckets/{new_bucket_id}")
|
| 495 |
+
huggingface_hub.copy_files(
|
| 496 |
+
f"hf://buckets/{bucket_id}/",
|
| 497 |
+
f"hf://buckets/{new_bucket_id}/",
|
| 498 |
+
)
|
| 499 |
+
inventory = bucket_inventory(new_bucket_id)
|
| 500 |
+
|
| 501 |
+
print(f"* Creating {new_space_id} with Trackio {trackio.__version__}")
|
| 502 |
+
deploy.deploy_as_space(new_space_id, bucket_id=new_bucket_id, private=private)
|
| 503 |
+
for key, variable in hf_api.get_space_variables(space_id).items():
|
| 504 |
+
if key not in _MANAGED_VARIABLES:
|
| 505 |
+
hf_api.add_space_variable(new_space_id, key, variable.value)
|
| 506 |
+
_wait_for_stage(new_space_id, frozenset({"RUNNING"}), timeout, hf_api)
|
| 507 |
+
verify_space_serves_inventory(new_space_id, inventory)
|
| 508 |
+
except Exception as e:
|
| 509 |
+
print(f"* Bump failed ({e}); removing {new_space_id} and its copied data")
|
| 510 |
+
try:
|
| 511 |
+
_remove_new_space(new_space_id, new_bucket_id, created_bucket, hf_api)
|
| 512 |
+
except Exception as cleanup_error:
|
| 513 |
+
print(f"* Could not remove {new_space_id}: {cleanup_error}")
|
| 514 |
+
raise BumpError(
|
| 515 |
+
f"Copying '{space_id}' into '{new_space_id}' failed and the new Space "
|
| 516 |
+
f"was removed; '{space_id}' was not modified: {e}"
|
| 517 |
+
) from e
|
| 518 |
+
print(
|
| 519 |
+
f"* {new_space_id} runs Trackio {trackio.__version__} with a copy of "
|
| 520 |
+
f"{space_id}'s data: {deploy.SPACE_URL.format(space_id=new_space_id)}"
|
| 521 |
+
)
|
| 522 |
+
return new_space_id
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
def _remove_new_space(
|
| 526 |
+
new_space_id: str,
|
| 527 |
+
new_bucket_id: str,
|
| 528 |
+
created_bucket: bool,
|
| 529 |
+
hf_api: huggingface_hub.HfApi,
|
| 530 |
+
) -> None:
|
| 531 |
+
hf_api.delete_repo(new_space_id, repo_type="space", missing_ok=True)
|
| 532 |
+
if created_bucket:
|
| 533 |
+
hf_api.delete_bucket(new_bucket_id, missing_ok=True)
|
| 534 |
+
return
|
| 535 |
+
copied = _list_bucket_file_paths(new_bucket_id)
|
| 536 |
+
if copied:
|
| 537 |
+
huggingface_hub.batch_bucket_files(new_bucket_id, delete=copied)
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
def bump(
|
| 541 |
+
space_id: str,
|
| 542 |
+
new_space_id: str | None = None,
|
| 543 |
+
new_bucket_id: str | None = None,
|
| 544 |
+
timeout: int = 1800,
|
| 545 |
+
) -> str:
|
| 546 |
+
"""
|
| 547 |
+
Upgrades a Trackio Space to the locally installed Trackio version.
|
| 548 |
+
|
| 549 |
+
Without `new_space_id`, the Space is upgraded in place: it is paused (clients
|
| 550 |
+
keep logging to the bucket inbox), its databases are backed up inside its
|
| 551 |
+
bucket, its Trackio pin is updated and it is restarted so the new server
|
| 552 |
+
migrates the data. If the Space fails to come back or loses data, the backup
|
| 553 |
+
and the previous Space files are restored.
|
| 554 |
+
|
| 555 |
+
With `new_space_id`, the source Space is left untouched and its bucket data is
|
| 556 |
+
copied into `new_bucket_id`, which backs a new Space running the local version.
|
| 557 |
+
|
| 558 |
+
Args:
|
| 559 |
+
space_id (`str`):
|
| 560 |
+
The Trackio Space to upgrade. Must run Trackio 0.21.0 or newer with a
|
| 561 |
+
bucket mounted at `/data`.
|
| 562 |
+
new_space_id (`str`, *optional*):
|
| 563 |
+
Create this Space from a copy of `space_id` instead of upgrading in place.
|
| 564 |
+
new_bucket_id (`str`, *optional*):
|
| 565 |
+
The bucket for `new_space_id`. Defaults to `{new_space_id}-bucket`.
|
| 566 |
+
timeout (`int`, *optional*, defaults to `1800`):
|
| 567 |
+
Seconds to wait for the upgraded Space to build and start.
|
| 568 |
+
|
| 569 |
+
Returns:
|
| 570 |
+
`str`: The ID of the Space now running the local Trackio version.
|
| 571 |
+
"""
|
| 572 |
+
if new_bucket_id is not None and new_space_id is None:
|
| 573 |
+
raise BumpError("new_bucket_id requires new_space_id.")
|
| 574 |
+
space_id, _, _ = preprocess_space_and_dataset_ids(space_id, None, None)
|
| 575 |
+
hf_api = huggingface_hub.HfApi()
|
| 576 |
+
try:
|
| 577 |
+
info = hf_api.space_info(space_id)
|
| 578 |
+
except RepositoryNotFoundError:
|
| 579 |
+
raise BumpError(f"Space '{space_id}' not found.")
|
| 580 |
+
if info.sdk != "gradio":
|
| 581 |
+
raise BumpError(
|
| 582 |
+
f"Space '{space_id}' is not a Trackio Gradio Space (sdk={info.sdk!r})."
|
| 583 |
+
)
|
| 584 |
+
bucket_id = deploy._get_existing_space_bucket(space_id, hf_api=hf_api)
|
| 585 |
+
if bucket_id is None:
|
| 586 |
+
raise BumpError(
|
| 587 |
+
f"Space '{space_id}' has no bucket mounted at '/data'. "
|
| 588 |
+
"`trackio bump` requires bucket storage."
|
| 589 |
+
)
|
| 590 |
+
space_version = _check_bumpable(
|
| 591 |
+
space_id, get_space_trackio_version(space_id, hf_api)
|
| 592 |
+
)
|
| 593 |
+
|
| 594 |
+
if new_space_id is not None:
|
| 595 |
+
new_space_id, _, auto_bucket_id = preprocess_space_and_dataset_ids(
|
| 596 |
+
new_space_id, None, new_bucket_id
|
| 597 |
+
)
|
| 598 |
+
return _bump_into_new_space(
|
| 599 |
+
space_id,
|
| 600 |
+
bucket_id,
|
| 601 |
+
new_space_id,
|
| 602 |
+
auto_bucket_id,
|
| 603 |
+
bool(info.private),
|
| 604 |
+
hf_api,
|
| 605 |
+
timeout,
|
| 606 |
+
)
|
| 607 |
+
|
| 608 |
+
if (
|
| 609 |
+
space_version == trackio.__version__
|
| 610 |
+
and not deploy._is_trackio_installed_from_source()
|
| 611 |
+
):
|
| 612 |
+
print(f"* {space_id} already runs Trackio {space_version}")
|
| 613 |
+
return space_id
|
| 614 |
+
return _bump_in_place(space_id, bucket_id, space_version, hf_api, timeout)
|
trackio/cas.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Content-addressed storage (CAS) helpers shared by the artifact client,
|
| 2 |
+
server, and storage layers.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
import os
|
| 7 |
+
import re
|
| 8 |
+
import uuid
|
| 9 |
+
from collections.abc import Iterable, Iterator
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
from trackio import utils
|
| 13 |
+
from trackio.typehints import Sha256Digest
|
| 14 |
+
|
| 15 |
+
SHA256_DIGEST_RE = re.compile(r"^[0-9a-f]{64}$")
|
| 16 |
+
ARTIFACT_VERSION_SPEC_RE = re.compile(r"^v(\d+)$")
|
| 17 |
+
ARTIFACT_NAME_RE = re.compile(r"^[A-Za-z0-9._-]+\Z")
|
| 18 |
+
HASH_CHUNK_SIZE = 1024 * 1024
|
| 19 |
+
PARTIAL_BLOB_MARKER = ".partial."
|
| 20 |
+
PARTIAL_BLOB_GLOB = f"*{PARTIAL_BLOB_MARKER}*"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def partial_blob_name(base: str = "") -> str:
|
| 24 |
+
"""Temp filename for an in-flight blob write, excluded from sync/export."""
|
| 25 |
+
return f"{base}{PARTIAL_BLOB_MARKER}{uuid.uuid4().hex}"
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def is_partial_blob(filename: str) -> bool:
|
| 29 |
+
return PARTIAL_BLOB_MARKER in filename
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def validate_artifact_name(name: str) -> str:
|
| 33 |
+
if not isinstance(name, str) or not ARTIFACT_NAME_RE.match(name):
|
| 34 |
+
raise ValueError(
|
| 35 |
+
f"Artifact name {name!r} must match ^[A-Za-z0-9._-]+$ "
|
| 36 |
+
"(letters, digits, dot, underscore, hyphen)."
|
| 37 |
+
)
|
| 38 |
+
return name
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def validate_aliases(aliases: list[str] | None) -> list[str]:
|
| 42 |
+
if aliases is None:
|
| 43 |
+
return []
|
| 44 |
+
if isinstance(aliases, str) or not isinstance(aliases, (list, tuple)):
|
| 45 |
+
raise ValueError(
|
| 46 |
+
f"aliases must be a list of strings, got {type(aliases).__name__}."
|
| 47 |
+
)
|
| 48 |
+
validated: list[str] = []
|
| 49 |
+
for alias in aliases:
|
| 50 |
+
if not isinstance(alias, str) or not alias:
|
| 51 |
+
raise ValueError(f"Alias {alias!r} must be a non-empty string.")
|
| 52 |
+
if not ARTIFACT_NAME_RE.match(alias):
|
| 53 |
+
raise ValueError(
|
| 54 |
+
f"Alias {alias!r} must match ^[A-Za-z0-9._-]+$ "
|
| 55 |
+
"(letters, digits, dot, underscore, hyphen)."
|
| 56 |
+
)
|
| 57 |
+
if ARTIFACT_VERSION_SPEC_RE.match(alias):
|
| 58 |
+
raise ValueError(
|
| 59 |
+
f"Alias {alias!r} is reserved for version pointers (vN); "
|
| 60 |
+
"choose another."
|
| 61 |
+
)
|
| 62 |
+
validated.append(alias)
|
| 63 |
+
return validated
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _project_blobs_root(project: str) -> Path:
|
| 67 |
+
return utils.project_artifacts_dir(project) / "blobs" / "sha256"
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def blob_path(project: str, digest: Sha256Digest) -> Path:
|
| 71 |
+
return _project_blobs_root(project) / digest[:2] / digest
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def hash_file(path: Path) -> tuple[Sha256Digest, int]:
|
| 75 |
+
size = path.stat().st_size
|
| 76 |
+
file_digest = getattr(hashlib, "file_digest", None)
|
| 77 |
+
if file_digest is not None:
|
| 78 |
+
with path.open("rb") as f:
|
| 79 |
+
sha = file_digest(f, "sha256")
|
| 80 |
+
else:
|
| 81 |
+
sha = hashlib.sha256()
|
| 82 |
+
with path.open("rb") as f:
|
| 83 |
+
while chunk := f.read(HASH_CHUNK_SIZE):
|
| 84 |
+
sha.update(chunk)
|
| 85 |
+
return Sha256Digest(sha.hexdigest()), size
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def validate_logical_path(logical: str) -> str:
|
| 89 |
+
"""Validate a manifest logical path: relative, no `.`/`..` segments, no
|
| 90 |
+
backslashes, no drive letters, non-empty. Returns the path unchanged.
|
| 91 |
+
"""
|
| 92 |
+
if not isinstance(logical, str) or not logical:
|
| 93 |
+
raise ValueError(f"Invalid artifact path: {logical!r} (must be non-empty)")
|
| 94 |
+
if "\\" in logical or "\x00" in logical:
|
| 95 |
+
raise ValueError(
|
| 96 |
+
f"Invalid artifact path {logical!r}: backslashes and NUL are not allowed"
|
| 97 |
+
)
|
| 98 |
+
if logical.startswith("/") or re.match(r"^[A-Za-z]:", logical):
|
| 99 |
+
raise ValueError(f"Invalid artifact path {logical!r}: must be relative")
|
| 100 |
+
parts = logical.split("/")
|
| 101 |
+
if any(part in ("", ".", "..") for part in parts):
|
| 102 |
+
raise ValueError(
|
| 103 |
+
f"Invalid artifact path {logical!r}: "
|
| 104 |
+
"'.', '..', and empty segments are not allowed"
|
| 105 |
+
)
|
| 106 |
+
return logical
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def assert_manifest_paths_compatible(paths: Iterable[str]) -> None:
|
| 110 |
+
"""Validate a manifest's logical paths as a set: reject exact duplicates and
|
| 111 |
+
file/directory prefix collisions. A path cannot be both a file and a
|
| 112 |
+
directory, so `sub` and `sub/x` (or `a/b` and `a/b/c`) cannot coexist;
|
| 113 |
+
siblings like `a/b` and `a/c` are fine.
|
| 114 |
+
"""
|
| 115 |
+
seen: set[str] = set()
|
| 116 |
+
for p in paths:
|
| 117 |
+
if p in seen:
|
| 118 |
+
raise ValueError(f"Duplicate logical path in manifest: {p!r}")
|
| 119 |
+
seen.add(p)
|
| 120 |
+
for p in seen:
|
| 121 |
+
parts = p.split("/")
|
| 122 |
+
for i in range(1, len(parts)):
|
| 123 |
+
ancestor = "/".join(parts[:i])
|
| 124 |
+
if ancestor in seen:
|
| 125 |
+
raise ValueError(
|
| 126 |
+
f"Artifact path {p!r} collides with {ancestor!r}: a path "
|
| 127 |
+
"cannot be both a file and a directory in the same artifact."
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def validate_digest(digest: str) -> Sha256Digest:
|
| 132 |
+
"""Validate that `digest` is a 64-char lowercase sha256 hex string and
|
| 133 |
+
return it unchanged.
|
| 134 |
+
"""
|
| 135 |
+
if not isinstance(digest, str) or not SHA256_DIGEST_RE.match(digest):
|
| 136 |
+
raise ValueError(f"Invalid artifact blob digest: {digest!r}")
|
| 137 |
+
return Sha256Digest(digest)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def stage_blob_from_chunks(
|
| 141 |
+
chunks: Iterable[bytes],
|
| 142 |
+
claimed_digest: Sha256Digest,
|
| 143 |
+
target_path: Path,
|
| 144 |
+
) -> None:
|
| 145 |
+
"""Stream `chunks` to `target_path`, verifying they hash to `claimed_digest`.
|
| 146 |
+
No-op if `target_path` already exists (the chunks are not consumed).
|
| 147 |
+
"""
|
| 148 |
+
if target_path.is_file():
|
| 149 |
+
return
|
| 150 |
+
target_path.parent.mkdir(parents=True, exist_ok=True)
|
| 151 |
+
partial = target_path.parent / partial_blob_name(target_path.name)
|
| 152 |
+
sha = hashlib.sha256()
|
| 153 |
+
try:
|
| 154 |
+
with partial.open("wb") as dst:
|
| 155 |
+
for chunk in chunks:
|
| 156 |
+
if not chunk:
|
| 157 |
+
continue
|
| 158 |
+
sha.update(chunk)
|
| 159 |
+
dst.write(chunk)
|
| 160 |
+
actual = sha.hexdigest()
|
| 161 |
+
if actual != claimed_digest:
|
| 162 |
+
raise ValueError(
|
| 163 |
+
f"Digest mismatch: claimed {claimed_digest}, computed {actual}"
|
| 164 |
+
)
|
| 165 |
+
os.replace(partial, target_path)
|
| 166 |
+
except Exception:
|
| 167 |
+
partial.unlink(missing_ok=True)
|
| 168 |
+
raise
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def stage_blob_from_file(
|
| 172 |
+
src_path: Path,
|
| 173 |
+
claimed_digest: Sha256Digest,
|
| 174 |
+
target_path: Path,
|
| 175 |
+
) -> None:
|
| 176 |
+
"""Copy `src_path` to `target_path`, verifying it hashes to `claimed_digest`."""
|
| 177 |
+
if target_path.is_file():
|
| 178 |
+
return
|
| 179 |
+
|
| 180 |
+
def _file_chunks() -> Iterator[bytes]:
|
| 181 |
+
with src_path.open("rb") as src:
|
| 182 |
+
while chunk := src.read(HASH_CHUNK_SIZE):
|
| 183 |
+
yield chunk
|
| 184 |
+
|
| 185 |
+
stage_blob_from_chunks(_file_chunks(), claimed_digest, target_path)
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def stage_blob_into_project(src_path: Path, project: str) -> tuple[Sha256Digest, int]:
|
| 189 |
+
"""Copy `src_path` into the project's CAS in a single pass, computing its
|
| 190 |
+
sha256 and size while writing. Returns `(digest, size)`. If the blob is
|
| 191 |
+
already present, the copy is discarded.
|
| 192 |
+
"""
|
| 193 |
+
blobs_root = _project_blobs_root(project)
|
| 194 |
+
blobs_root.mkdir(parents=True, exist_ok=True)
|
| 195 |
+
partial = blobs_root / partial_blob_name()
|
| 196 |
+
sha = hashlib.sha256()
|
| 197 |
+
size = 0
|
| 198 |
+
try:
|
| 199 |
+
with src_path.open("rb") as src, partial.open("wb") as dst:
|
| 200 |
+
while chunk := src.read(HASH_CHUNK_SIZE):
|
| 201 |
+
sha.update(chunk)
|
| 202 |
+
dst.write(chunk)
|
| 203 |
+
size += len(chunk)
|
| 204 |
+
digest = Sha256Digest(sha.hexdigest())
|
| 205 |
+
target = blob_path(project, digest)
|
| 206 |
+
if target.is_file():
|
| 207 |
+
partial.unlink(missing_ok=True)
|
| 208 |
+
return digest, size
|
| 209 |
+
target.parent.mkdir(parents=True, exist_ok=True)
|
| 210 |
+
os.replace(partial, target)
|
| 211 |
+
return digest, size
|
| 212 |
+
except Exception:
|
| 213 |
+
partial.unlink(missing_ok=True)
|
| 214 |
+
raise
|
trackio/cli.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
trackio/cli_helpers.py
ADDED
|
@@ -0,0 +1,673 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import re
|
| 3 |
+
import sys
|
| 4 |
+
from datetime import datetime, timezone
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
from trackio import references
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def format_json(data: Any) -> str:
|
| 11 |
+
"""Format data as JSON."""
|
| 12 |
+
return json.dumps(data, indent=2)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def format_list(items: list[str], title: str | None = None) -> str:
|
| 16 |
+
"""Format a list of items in human-readable format."""
|
| 17 |
+
if not items:
|
| 18 |
+
return f"No {title.lower() if title else 'items'} found."
|
| 19 |
+
|
| 20 |
+
output = []
|
| 21 |
+
if title:
|
| 22 |
+
output.append(f"{title}:")
|
| 23 |
+
|
| 24 |
+
for item in items:
|
| 25 |
+
output.append(f" - {item}")
|
| 26 |
+
|
| 27 |
+
return "\n".join(output)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def format_artifacts(artifacts: list[dict], project: str | None = None) -> str:
|
| 31 |
+
"""Format a project's artifacts in human-readable format."""
|
| 32 |
+
if not artifacts:
|
| 33 |
+
return "No artifacts found."
|
| 34 |
+
|
| 35 |
+
title = f"Artifacts in '{project}'" if project else "Artifacts"
|
| 36 |
+
output = [f"{title}:"]
|
| 37 |
+
for art in artifacts:
|
| 38 |
+
latest = art.get("latest_version")
|
| 39 |
+
version = f"v{latest}" if latest is not None else "(no versions)"
|
| 40 |
+
aliases = art.get("aliases") or []
|
| 41 |
+
alias_str = f" [{', '.join(aliases)}]" if aliases else ""
|
| 42 |
+
output.append(
|
| 43 |
+
f" - {art['name']} ({art.get('type', '?')}) "
|
| 44 |
+
f"latest={version}{alias_str} "
|
| 45 |
+
f"versions={art.get('num_versions', 0)}"
|
| 46 |
+
)
|
| 47 |
+
if art.get("description"):
|
| 48 |
+
output.append(f" {art['description']}")
|
| 49 |
+
return "\n".join(output)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def format_artifact(record: dict) -> str:
|
| 53 |
+
"""Format a single resolved artifact version in human-readable format."""
|
| 54 |
+
lines = [
|
| 55 |
+
f"Artifact: {record['name']} (v{record['version']})",
|
| 56 |
+
f" type: {record.get('type')}",
|
| 57 |
+
]
|
| 58 |
+
if record.get("description"):
|
| 59 |
+
lines.append(f" description: {record['description']}")
|
| 60 |
+
aliases = record.get("aliases") or []
|
| 61 |
+
if aliases:
|
| 62 |
+
lines.append(f" aliases: {', '.join(aliases)}")
|
| 63 |
+
if record.get("metadata"):
|
| 64 |
+
lines.append(f" metadata: {record['metadata']}")
|
| 65 |
+
lines.append(f" size: {record.get('size_bytes')} bytes")
|
| 66 |
+
lines.append(f" digest: {record.get('manifest_digest')}")
|
| 67 |
+
manifest = record.get("manifest") or []
|
| 68 |
+
lines.append(f" files ({len(manifest)}):")
|
| 69 |
+
for entry in manifest:
|
| 70 |
+
if references.is_reference_entry(entry):
|
| 71 |
+
lines.append(f" - {entry['path']} -> {entry['ref']} (reference)")
|
| 72 |
+
else:
|
| 73 |
+
lines.append(f" - {entry['path']} ({entry['size']} bytes)")
|
| 74 |
+
return "\n".join(lines)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def format_registry_collections(registry: str, collections: list[dict]) -> str:
|
| 78 |
+
"""Format the collections in an artifact registry."""
|
| 79 |
+
if not collections:
|
| 80 |
+
return f"No collections found in registry '{registry}'."
|
| 81 |
+
|
| 82 |
+
output = [f"Collections in registry '{registry}':"]
|
| 83 |
+
for collection in collections:
|
| 84 |
+
latest = collection.get("latest_version")
|
| 85 |
+
latest_display = f"v{latest}" if latest is not None else "(empty)"
|
| 86 |
+
output.append(
|
| 87 |
+
f" - {collection['name']} ({collection['type']}) "
|
| 88 |
+
f"latest={latest_display} versions={collection['num_links']}"
|
| 89 |
+
)
|
| 90 |
+
if collection.get("description"):
|
| 91 |
+
output.append(f" {collection['description']}")
|
| 92 |
+
return "\n".join(output)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def format_registry_collection(registry: str, collection: dict) -> str:
|
| 96 |
+
"""Format one registry collection and its linked versions."""
|
| 97 |
+
output = [
|
| 98 |
+
f"Collection: {registry}/{collection['name']}",
|
| 99 |
+
f" type: {collection['type']}",
|
| 100 |
+
f" versions: {collection['num_links']}",
|
| 101 |
+
]
|
| 102 |
+
if collection.get("description"):
|
| 103 |
+
output.append(f" description: {collection['description']}")
|
| 104 |
+
if not collection["links"]:
|
| 105 |
+
output.append(" No linked versions.")
|
| 106 |
+
return "\n".join(output)
|
| 107 |
+
|
| 108 |
+
output.append(" Linked versions:")
|
| 109 |
+
for link in collection["links"]:
|
| 110 |
+
aliases = link.get("aliases") or []
|
| 111 |
+
alias_display = f" [{', '.join(aliases)}]" if aliases else ""
|
| 112 |
+
output.append(
|
| 113 |
+
f" - v{link['collection_version']} -> "
|
| 114 |
+
f"{link['source_project']}/{link['source_artifact']}"
|
| 115 |
+
f":v{link['source_version']}{alias_display}"
|
| 116 |
+
)
|
| 117 |
+
return "\n".join(output)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def format_registry_events(events: list[dict]) -> str:
|
| 121 |
+
"""Format a registry audit log."""
|
| 122 |
+
if not events:
|
| 123 |
+
return "No registry events found."
|
| 124 |
+
return "\n".join(
|
| 125 |
+
f"{event['ts']} | {event['kind']} | "
|
| 126 |
+
f"{json.dumps(event['payload'], sort_keys=True)}"
|
| 127 |
+
for event in events
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def format_spaces(spaces: list[dict]) -> str:
|
| 132 |
+
"""Format HF Spaces in human-readable format."""
|
| 133 |
+
if not spaces:
|
| 134 |
+
return "No Trackio Spaces found."
|
| 135 |
+
|
| 136 |
+
output = ["Trackio Spaces:"]
|
| 137 |
+
for space in spaces:
|
| 138 |
+
visibility = "private" if space.get("private") else "public"
|
| 139 |
+
output.append(f" - {space['id']} ({visibility})")
|
| 140 |
+
if space.get("url"):
|
| 141 |
+
output.append(f" {space['url']}")
|
| 142 |
+
|
| 143 |
+
return "\n".join(output)
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def format_project_summary(summary: dict) -> str:
|
| 147 |
+
"""Format project summary in human-readable format."""
|
| 148 |
+
output = [f"Project: {summary['project']}"]
|
| 149 |
+
output.append(f"Number of runs: {summary['num_runs']}")
|
| 150 |
+
|
| 151 |
+
if summary["runs"]:
|
| 152 |
+
output.append("\nRuns:")
|
| 153 |
+
for run in summary["runs"]:
|
| 154 |
+
output.append(f" - {run}")
|
| 155 |
+
else:
|
| 156 |
+
output.append("\nNo runs found.")
|
| 157 |
+
|
| 158 |
+
if summary.get("last_activity"):
|
| 159 |
+
output.append(f"\nLast activity (max step): {summary['last_activity']}")
|
| 160 |
+
|
| 161 |
+
return "\n".join(output)
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def format_run_summary(summary: dict) -> str:
|
| 165 |
+
"""Format run summary in human-readable format."""
|
| 166 |
+
output = [f"Project: {summary['project']}"]
|
| 167 |
+
output.append(f"Run: {summary['run']}")
|
| 168 |
+
output.append(f"Number of logs: {summary['num_logs']}")
|
| 169 |
+
|
| 170 |
+
if summary.get("last_step") is not None:
|
| 171 |
+
output.append(f"Last step: {summary['last_step']}")
|
| 172 |
+
|
| 173 |
+
if summary.get("metrics"):
|
| 174 |
+
output.append("\nMetrics:")
|
| 175 |
+
for metric in summary["metrics"]:
|
| 176 |
+
output.append(f" - {metric}")
|
| 177 |
+
else:
|
| 178 |
+
output.append("\nNo metrics found.")
|
| 179 |
+
|
| 180 |
+
config = summary.get("config")
|
| 181 |
+
if config:
|
| 182 |
+
output.append("\nConfig:")
|
| 183 |
+
config_display = {k: v for k, v in config.items() if not k.startswith("_")}
|
| 184 |
+
if config_display:
|
| 185 |
+
for key, value in config_display.items():
|
| 186 |
+
output.append(f" {key}: {value}")
|
| 187 |
+
else:
|
| 188 |
+
output.append(" (no config)")
|
| 189 |
+
else:
|
| 190 |
+
output.append("\nConfig: (no config)")
|
| 191 |
+
|
| 192 |
+
return "\n".join(output)
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def format_metric_values(values: list[dict]) -> str:
|
| 196 |
+
"""Format metric values in human-readable format."""
|
| 197 |
+
if not values:
|
| 198 |
+
return "No metric values found."
|
| 199 |
+
|
| 200 |
+
output = [f"Found {len(values)} value(s):\n"]
|
| 201 |
+
output.append("Step | Timestamp | Value")
|
| 202 |
+
output.append("-" * 50)
|
| 203 |
+
|
| 204 |
+
for value in values:
|
| 205 |
+
step = value.get("step", "N/A")
|
| 206 |
+
timestamp = value.get("timestamp", "N/A")
|
| 207 |
+
val = value.get("value", "N/A")
|
| 208 |
+
output.append(f"{step} | {timestamp} | {val}")
|
| 209 |
+
|
| 210 |
+
return "\n".join(output)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def format_system_metrics(metrics: list[dict]) -> str:
|
| 214 |
+
"""Format system metrics in human-readable format."""
|
| 215 |
+
if not metrics:
|
| 216 |
+
return "No system metrics found."
|
| 217 |
+
|
| 218 |
+
output = [f"Found {len(metrics)} system metric entry/entries:\n"]
|
| 219 |
+
|
| 220 |
+
for i, entry in enumerate(metrics):
|
| 221 |
+
timestamp = entry.get("timestamp", "N/A")
|
| 222 |
+
output.append(f"\nEntry {i + 1} (Timestamp: {timestamp}):")
|
| 223 |
+
for key, value in entry.items():
|
| 224 |
+
if key != "timestamp":
|
| 225 |
+
output.append(f" {key}: {value}")
|
| 226 |
+
|
| 227 |
+
return "\n".join(output)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def format_system_metric_names(names: list[str]) -> str:
|
| 231 |
+
"""Format system metric names in human-readable format."""
|
| 232 |
+
return format_list(names, "System Metrics")
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def format_snapshot(snapshot: dict[str, list[dict]]) -> str:
|
| 236 |
+
"""Format a metrics snapshot in human-readable format."""
|
| 237 |
+
if not snapshot:
|
| 238 |
+
return "No metrics found in the specified range."
|
| 239 |
+
|
| 240 |
+
output = []
|
| 241 |
+
for metric_name, values in sorted(snapshot.items()):
|
| 242 |
+
output.append(f"\n{metric_name}:")
|
| 243 |
+
output.append(" Step | Timestamp | Value")
|
| 244 |
+
output.append(" " + "-" * 48)
|
| 245 |
+
for v in values:
|
| 246 |
+
step = v.get("step", "N/A")
|
| 247 |
+
ts = v.get("timestamp", "N/A")
|
| 248 |
+
val = v.get("value", "N/A")
|
| 249 |
+
output.append(f" {step} | {ts} | {val}")
|
| 250 |
+
|
| 251 |
+
return "\n".join(output)
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def format_alerts(alerts: list[dict]) -> str:
|
| 255 |
+
"""Format alerts in human-readable format."""
|
| 256 |
+
if not alerts:
|
| 257 |
+
return "No alerts found."
|
| 258 |
+
|
| 259 |
+
output = [f"Found {len(alerts)} alert(s):\n"]
|
| 260 |
+
output.append("Timestamp | Run | Level | Title | Text | Step")
|
| 261 |
+
output.append("-" * 80)
|
| 262 |
+
|
| 263 |
+
for a in alerts:
|
| 264 |
+
ts = a.get("timestamp", "N/A")
|
| 265 |
+
run = a.get("run", "N/A")
|
| 266 |
+
level = a.get("level", "N/A").upper()
|
| 267 |
+
title = a.get("title", "")
|
| 268 |
+
text = a.get("text", "") or ""
|
| 269 |
+
step = a.get("step", "N/A")
|
| 270 |
+
output.append(f"{ts} | {run} | {level} | {title} | {text} | {step}")
|
| 271 |
+
|
| 272 |
+
return "\n".join(output)
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def format_query_result(result: dict[str, Any]) -> str:
|
| 276 |
+
"""Format SQL query results in human-readable format."""
|
| 277 |
+
columns = result.get("columns", [])
|
| 278 |
+
rows = result.get("rows", [])
|
| 279 |
+
row_count = result.get("row_count", 0)
|
| 280 |
+
|
| 281 |
+
if not columns:
|
| 282 |
+
return f"Query returned {row_count} row(s)."
|
| 283 |
+
|
| 284 |
+
rendered_rows = []
|
| 285 |
+
for row in rows:
|
| 286 |
+
rendered_rows.append(
|
| 287 |
+
[
|
| 288 |
+
"" if row.get(column) is None else str(row.get(column))
|
| 289 |
+
for column in columns
|
| 290 |
+
]
|
| 291 |
+
)
|
| 292 |
+
|
| 293 |
+
widths = []
|
| 294 |
+
for idx, column in enumerate(columns):
|
| 295 |
+
cell_width = max(
|
| 296 |
+
(len(rendered_row[idx]) for rendered_row in rendered_rows), default=0
|
| 297 |
+
)
|
| 298 |
+
widths.append(max(len(column), cell_width))
|
| 299 |
+
|
| 300 |
+
header = " | ".join(
|
| 301 |
+
column.ljust(width) for column, width in zip(columns, widths, strict=False)
|
| 302 |
+
)
|
| 303 |
+
separator = "-+-".join("-" * width for width in widths)
|
| 304 |
+
output = [f"Query returned {row_count} row(s).", header, separator]
|
| 305 |
+
|
| 306 |
+
if not rendered_rows:
|
| 307 |
+
output.append("(no rows)")
|
| 308 |
+
return "\n".join(output)
|
| 309 |
+
|
| 310 |
+
for rendered_row in rendered_rows:
|
| 311 |
+
output.append(
|
| 312 |
+
" | ".join(
|
| 313 |
+
value.ljust(width)
|
| 314 |
+
for value, width in zip(rendered_row, widths, strict=False)
|
| 315 |
+
)
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
return "\n".join(output)
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def error_exit(message: str, code: int = 1) -> None:
|
| 322 |
+
"""Print error message and exit."""
|
| 323 |
+
print(f"Error: {message}", file=sys.stderr)
|
| 324 |
+
sys.exit(code)
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
def format_duration_ms(value: Any) -> str:
|
| 328 |
+
"""Render a millisecond duration the way the trace dashboard does."""
|
| 329 |
+
if value is None or value == "":
|
| 330 |
+
return "-"
|
| 331 |
+
try:
|
| 332 |
+
ms = float(value)
|
| 333 |
+
except (TypeError, ValueError):
|
| 334 |
+
return "-"
|
| 335 |
+
if ms < 1000:
|
| 336 |
+
return f"{round(ms)}ms"
|
| 337 |
+
if ms < 60_000:
|
| 338 |
+
return f"{ms / 1000:.2f}s" if ms < 10_000 else f"{ms / 1000:.1f}s"
|
| 339 |
+
minutes = int(ms // 60_000)
|
| 340 |
+
seconds = round((ms % 60_000) / 1000)
|
| 341 |
+
return f"{minutes}m {seconds}s"
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
def format_cost_usd(value: Any) -> str:
|
| 345 |
+
if value is None or value == "":
|
| 346 |
+
return "-"
|
| 347 |
+
try:
|
| 348 |
+
cost = float(value)
|
| 349 |
+
except (TypeError, ValueError):
|
| 350 |
+
return "-"
|
| 351 |
+
if cost == 0:
|
| 352 |
+
return "$0"
|
| 353 |
+
if cost < 0.0001:
|
| 354 |
+
return "<$0.0001"
|
| 355 |
+
if cost < 0.01:
|
| 356 |
+
return f"${cost:.4f}"
|
| 357 |
+
return f"${cost:.2f}"
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
_ISO_FRACTION_RE = re.compile(r"\.(\d+)")
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
def parse_span_timestamp(value: Any) -> datetime | None:
|
| 364 |
+
"""Parse a span timestamp, tolerating any number of fractional digits.
|
| 365 |
+
|
| 366 |
+
`datetime.fromisoformat` accepts only 3 or 6 fractional digits before
|
| 367 |
+
Python 3.11, and instrumentation commonly emits fewer (or a `Z` suffix),
|
| 368 |
+
so those are normalized first. Naive timestamps are assumed to be UTC so
|
| 369 |
+
that spans from mixed sources stay comparable.
|
| 370 |
+
"""
|
| 371 |
+
if not value:
|
| 372 |
+
return None
|
| 373 |
+
text = str(value).strip()
|
| 374 |
+
if text.endswith(("Z", "z")):
|
| 375 |
+
text = text[:-1] + "+00:00"
|
| 376 |
+
text = _ISO_FRACTION_RE.sub(
|
| 377 |
+
lambda match: "." + (match.group(1) + "000000")[:6], text, count=1
|
| 378 |
+
)
|
| 379 |
+
try:
|
| 380 |
+
parsed = datetime.fromisoformat(text)
|
| 381 |
+
except ValueError:
|
| 382 |
+
return None
|
| 383 |
+
return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
def _span_duration_ms(span: dict) -> float | None:
|
| 387 |
+
explicit = span.get("duration_ms")
|
| 388 |
+
if explicit is not None:
|
| 389 |
+
try:
|
| 390 |
+
return max(0.0, float(explicit))
|
| 391 |
+
except (TypeError, ValueError):
|
| 392 |
+
pass
|
| 393 |
+
started = parse_span_timestamp(span.get("start_time"))
|
| 394 |
+
ended = parse_span_timestamp(span.get("end_time"))
|
| 395 |
+
if started is None or ended is None:
|
| 396 |
+
return None
|
| 397 |
+
return max(0.0, (ended - started).total_seconds() * 1000)
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def _span_is_error(span: dict) -> bool:
|
| 401 |
+
status = str(span.get("status") or "").lower()
|
| 402 |
+
return status in {"error", "failed"} or bool(span.get("error"))
|
| 403 |
+
|
| 404 |
+
|
| 405 |
+
def trace_rollup(trace: dict) -> dict:
|
| 406 |
+
"""Latency, cost, token, and status totals for one trace."""
|
| 407 |
+
spans = trace.get("spans") or []
|
| 408 |
+
starts, ends, durations = [], [], []
|
| 409 |
+
cost = 0.0
|
| 410 |
+
has_cost = False
|
| 411 |
+
input_tokens = output_tokens = 0
|
| 412 |
+
has_usage = has_error = False
|
| 413 |
+
|
| 414 |
+
for span in spans:
|
| 415 |
+
if not isinstance(span, dict):
|
| 416 |
+
continue
|
| 417 |
+
for field, sink in (("start_time", starts), ("end_time", ends)):
|
| 418 |
+
parsed = parse_span_timestamp(span.get(field))
|
| 419 |
+
if parsed is not None:
|
| 420 |
+
sink.append(parsed)
|
| 421 |
+
duration = _span_duration_ms(span)
|
| 422 |
+
if duration is not None:
|
| 423 |
+
durations.append(duration)
|
| 424 |
+
span_cost = span.get("cost_usd")
|
| 425 |
+
if span_cost is not None:
|
| 426 |
+
try:
|
| 427 |
+
cost += float(span_cost)
|
| 428 |
+
has_cost = True
|
| 429 |
+
except (TypeError, ValueError):
|
| 430 |
+
pass
|
| 431 |
+
usage = span.get("usage") or {}
|
| 432 |
+
for key, add in (("input_tokens", "in"), ("output_tokens", "out")):
|
| 433 |
+
value = usage.get(key)
|
| 434 |
+
if value is None:
|
| 435 |
+
continue
|
| 436 |
+
try:
|
| 437 |
+
if add == "in":
|
| 438 |
+
input_tokens += int(value)
|
| 439 |
+
else:
|
| 440 |
+
output_tokens += int(value)
|
| 441 |
+
has_usage = True
|
| 442 |
+
except (TypeError, ValueError):
|
| 443 |
+
pass
|
| 444 |
+
if _span_is_error(span):
|
| 445 |
+
has_error = True
|
| 446 |
+
|
| 447 |
+
metadata = trace.get("metadata") or {}
|
| 448 |
+
if starts and ends:
|
| 449 |
+
duration_ms = max(0.0, (max(ends) - min(starts)).total_seconds() * 1000)
|
| 450 |
+
elif durations:
|
| 451 |
+
duration_ms = max(durations)
|
| 452 |
+
else:
|
| 453 |
+
duration_ms = metadata.get("duration_ms") or metadata.get("latency_ms")
|
| 454 |
+
|
| 455 |
+
all_success = bool(spans) and all(
|
| 456 |
+
str(span.get("status") or "").lower() == "success"
|
| 457 |
+
for span in spans
|
| 458 |
+
if isinstance(span, dict)
|
| 459 |
+
)
|
| 460 |
+
metadata_status = metadata.get("status")
|
| 461 |
+
status = (
|
| 462 |
+
"error"
|
| 463 |
+
if has_error
|
| 464 |
+
else (
|
| 465 |
+
metadata_status
|
| 466 |
+
if isinstance(metadata_status, str) and metadata_status
|
| 467 |
+
else ("success" if all_success else None)
|
| 468 |
+
)
|
| 469 |
+
)
|
| 470 |
+
return {
|
| 471 |
+
"duration_ms": duration_ms,
|
| 472 |
+
"cost_usd": cost if has_cost else metadata.get("cost_usd"),
|
| 473 |
+
"input_tokens": input_tokens if has_usage else None,
|
| 474 |
+
"output_tokens": output_tokens if has_usage else None,
|
| 475 |
+
"status": status,
|
| 476 |
+
"span_count": len(spans),
|
| 477 |
+
}
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
def short_trace_id(trace_id: Any) -> str:
|
| 481 |
+
"""Shorten a stored trace id for display, as the dashboard does.
|
| 482 |
+
|
| 483 |
+
Stored ids are `run_id:log_id:key[:index]`; the `log_id` (plus the list
|
| 484 |
+
index, when the key held several traces) identifies the trace on its own.
|
| 485 |
+
"""
|
| 486 |
+
text = str(trace_id or "")
|
| 487 |
+
parts = text.split(":")
|
| 488 |
+
if len(parts) < 3:
|
| 489 |
+
return text
|
| 490 |
+
short = parts[1]
|
| 491 |
+
last = parts[-1]
|
| 492 |
+
if len(parts) >= 4 and last.lstrip("-").isdigit():
|
| 493 |
+
return f"{short}:{last}"
|
| 494 |
+
return short
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def trace_id_matches(trace: dict, wanted: str) -> bool:
|
| 498 |
+
stored = str(trace.get("id") or "")
|
| 499 |
+
return wanted in (stored, short_trace_id(stored))
|
| 500 |
+
|
| 501 |
+
|
| 502 |
+
def _first_message_text(trace: dict, role: str) -> str:
|
| 503 |
+
for message in trace.get("messages") or []:
|
| 504 |
+
if not isinstance(message, dict) or message.get("role") != role:
|
| 505 |
+
continue
|
| 506 |
+
content = message.get("content")
|
| 507 |
+
if isinstance(content, str):
|
| 508 |
+
return content
|
| 509 |
+
if isinstance(content, list):
|
| 510 |
+
for part in content:
|
| 511 |
+
if isinstance(part, dict) and isinstance(part.get("text"), str):
|
| 512 |
+
return part["text"]
|
| 513 |
+
return ""
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
def format_traces(traces: list[dict]) -> str:
|
| 517 |
+
"""Format a trace index in human-readable format."""
|
| 518 |
+
if not traces:
|
| 519 |
+
return "No traces found."
|
| 520 |
+
|
| 521 |
+
output = [f"Found {len(traces)} trace(s):\n"]
|
| 522 |
+
output.append("ID | Run | Step | Status | Latency | Cost | Ops | Request")
|
| 523 |
+
output.append("-" * 100)
|
| 524 |
+
for trace in traces:
|
| 525 |
+
rollup = trace_rollup(trace)
|
| 526 |
+
request = _first_message_text(trace, "user").replace("\n", " ")
|
| 527 |
+
if len(request) > 48:
|
| 528 |
+
request = request[:45] + "..."
|
| 529 |
+
output.append(
|
| 530 |
+
" | ".join(
|
| 531 |
+
[
|
| 532 |
+
short_trace_id(trace.get("id")) or "-",
|
| 533 |
+
str(trace.get("run", "-")),
|
| 534 |
+
str(trace.get("step", "-")),
|
| 535 |
+
rollup["status"] or "-",
|
| 536 |
+
format_duration_ms(rollup["duration_ms"]),
|
| 537 |
+
format_cost_usd(rollup["cost_usd"]),
|
| 538 |
+
str(rollup["span_count"]),
|
| 539 |
+
request or "-",
|
| 540 |
+
]
|
| 541 |
+
)
|
| 542 |
+
)
|
| 543 |
+
return "\n".join(output)
|
| 544 |
+
|
| 545 |
+
|
| 546 |
+
def _ordered_span_tree(spans: list[dict]) -> list[tuple[int, dict]]:
|
| 547 |
+
"""Depth-annotated spans, parents before children, siblings by start time."""
|
| 548 |
+
normalized = [span for span in spans if isinstance(span, dict)]
|
| 549 |
+
by_id: dict[str, dict] = {}
|
| 550 |
+
for index, span in enumerate(normalized):
|
| 551 |
+
span_id = str(span.get("id") or f"span-{index + 1}")
|
| 552 |
+
by_id.setdefault(span_id, span)
|
| 553 |
+
|
| 554 |
+
children: dict[int, list[dict]] = {}
|
| 555 |
+
roots: list[dict] = []
|
| 556 |
+
for span in normalized:
|
| 557 |
+
parent_id = span.get("parent_id")
|
| 558 |
+
parent = by_id.get(str(parent_id)) if parent_id else None
|
| 559 |
+
if parent is None or parent is span:
|
| 560 |
+
roots.append(span)
|
| 561 |
+
else:
|
| 562 |
+
children.setdefault(id(parent), []).append(span)
|
| 563 |
+
|
| 564 |
+
def sort_key(span: dict) -> tuple:
|
| 565 |
+
started = parse_span_timestamp(span.get("start_time"))
|
| 566 |
+
return (
|
| 567 |
+
0 if started else 1,
|
| 568 |
+
started.timestamp() if started else 0.0,
|
| 569 |
+
normalized.index(span),
|
| 570 |
+
)
|
| 571 |
+
|
| 572 |
+
rows: list[tuple[int, dict]] = []
|
| 573 |
+
seen: set[int] = set()
|
| 574 |
+
|
| 575 |
+
def walk(span: dict, depth: int) -> None:
|
| 576 |
+
if id(span) in seen:
|
| 577 |
+
return
|
| 578 |
+
seen.add(id(span))
|
| 579 |
+
rows.append((depth, span))
|
| 580 |
+
for child in sorted(children.get(id(span), []), key=sort_key):
|
| 581 |
+
walk(child, depth + 1)
|
| 582 |
+
|
| 583 |
+
for root in sorted(roots, key=sort_key):
|
| 584 |
+
walk(root, 0)
|
| 585 |
+
for span in normalized:
|
| 586 |
+
walk(span, 0)
|
| 587 |
+
return rows
|
| 588 |
+
|
| 589 |
+
|
| 590 |
+
def format_trace(trace: dict) -> str:
|
| 591 |
+
"""Format a single trace, including its span tree, in human-readable format."""
|
| 592 |
+
rollup = trace_rollup(trace)
|
| 593 |
+
output = [f"Trace {short_trace_id(trace.get('id')) or '-'}"]
|
| 594 |
+
output.append(f" Full id: {trace.get('id', '-')}")
|
| 595 |
+
output.append(f" Run: {trace.get('run', '-')}")
|
| 596 |
+
output.append(f" Step: {trace.get('step', '-')}")
|
| 597 |
+
output.append(
|
| 598 |
+
f" Logged: {trace.get('timestamp', '-')} (key: {trace.get('key', '-')})"
|
| 599 |
+
)
|
| 600 |
+
output.append(
|
| 601 |
+
f" Totals: {rollup['status'] or 'no status'} | "
|
| 602 |
+
f"{format_duration_ms(rollup['duration_ms'])} | "
|
| 603 |
+
f"{format_cost_usd(rollup['cost_usd'])} | "
|
| 604 |
+
f"{rollup['input_tokens'] if rollup['input_tokens'] is not None else '-'} in / "
|
| 605 |
+
f"{rollup['output_tokens'] if rollup['output_tokens'] is not None else '-'} out tokens | "
|
| 606 |
+
f"{rollup['span_count']} operation(s)"
|
| 607 |
+
)
|
| 608 |
+
metadata = trace.get("metadata") or {}
|
| 609 |
+
if metadata:
|
| 610 |
+
rendered = ", ".join(
|
| 611 |
+
f"{key}={value}"
|
| 612 |
+
for key, value in metadata.items()
|
| 613 |
+
if not isinstance(value, (dict, list))
|
| 614 |
+
)
|
| 615 |
+
if rendered:
|
| 616 |
+
output.append(f" Metadata: {rendered}")
|
| 617 |
+
|
| 618 |
+
spans = trace.get("spans") or []
|
| 619 |
+
if spans:
|
| 620 |
+
output.append("\nExecution:")
|
| 621 |
+
for depth, span in _ordered_span_tree(spans):
|
| 622 |
+
indent = " " * (depth + 1)
|
| 623 |
+
bits = [format_duration_ms(_span_duration_ms(span))]
|
| 624 |
+
if span.get("model"):
|
| 625 |
+
bits.append(str(span["model"]))
|
| 626 |
+
usage = span.get("usage") or {}
|
| 627 |
+
if (
|
| 628 |
+
usage.get("input_tokens") is not None
|
| 629 |
+
or usage.get("output_tokens") is not None
|
| 630 |
+
):
|
| 631 |
+
bits.append(
|
| 632 |
+
f"{usage.get('input_tokens', 0)}->{usage.get('output_tokens', 0)} tok"
|
| 633 |
+
)
|
| 634 |
+
if span.get("cost_usd") is not None:
|
| 635 |
+
bits.append(format_cost_usd(span.get("cost_usd")))
|
| 636 |
+
marker = "ERROR " if _span_is_error(span) else ""
|
| 637 |
+
output.append(
|
| 638 |
+
f"{indent}{marker}{span.get('name', span.get('kind', 'span'))} "
|
| 639 |
+
f"[{span.get('kind', 'span')}] {' '.join(bits)}"
|
| 640 |
+
)
|
| 641 |
+
error = span.get("error")
|
| 642 |
+
if error:
|
| 643 |
+
detail = error if isinstance(error, str) else json.dumps(error)
|
| 644 |
+
output.append(f"{indent} -> {detail[:160]}")
|
| 645 |
+
else:
|
| 646 |
+
output.append("\nExecution: no spans logged for this trace.")
|
| 647 |
+
|
| 648 |
+
messages = trace.get("messages") or []
|
| 649 |
+
if messages:
|
| 650 |
+
output.append("\nConversation:")
|
| 651 |
+
for message in messages:
|
| 652 |
+
if not isinstance(message, dict):
|
| 653 |
+
continue
|
| 654 |
+
role = message.get("role", "message")
|
| 655 |
+
content = message.get("content")
|
| 656 |
+
if not isinstance(content, str):
|
| 657 |
+
content = json.dumps(content) if content is not None else ""
|
| 658 |
+
content = content.replace("\n", " ")
|
| 659 |
+
if len(content) > 160:
|
| 660 |
+
content = content[:157] + "..."
|
| 661 |
+
output.append(f" {role}: {content}")
|
| 662 |
+
return "\n".join(output)
|
| 663 |
+
|
| 664 |
+
|
| 665 |
+
def format_trace_summary(result: dict[str, Any]) -> str:
|
| 666 |
+
"""Format the per-operation trace rollup in human-readable format."""
|
| 667 |
+
rows = result.get("rows") or []
|
| 668 |
+
if not rows:
|
| 669 |
+
return "No trace spans found."
|
| 670 |
+
columns = result.get("columns") or []
|
| 671 |
+
return format_query_result(
|
| 672 |
+
{"columns": columns, "rows": rows, "row_count": len(rows)}
|
| 673 |
+
)
|
trackio/commit_scheduler.py
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Originally copied from https://github.com/huggingface/huggingface_hub/blob/d0a948fc2a32ed6e557042a95ef3e4af97ec4a7c/src/huggingface_hub/_commit_scheduler.py
|
| 2 |
+
|
| 3 |
+
import atexit
|
| 4 |
+
import logging
|
| 5 |
+
import time
|
| 6 |
+
from concurrent.futures import Future
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from threading import Lock, Thread
|
| 10 |
+
from typing import Callable, Dict, List, Union
|
| 11 |
+
|
| 12 |
+
from huggingface_hub import CommitOperationAdd
|
| 13 |
+
from huggingface_hub.hf_api import DEFAULT_IGNORE_PATTERNS, CommitInfo, HfApi
|
| 14 |
+
from huggingface_hub.utils import filter_repo_objects
|
| 15 |
+
|
| 16 |
+
logger = logging.getLogger(__name__)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass(frozen=True)
|
| 20 |
+
class _FileToUpload:
|
| 21 |
+
"""Temporary dataclass to store info about files to upload. Not meant to be used directly."""
|
| 22 |
+
|
| 23 |
+
local_path: Path
|
| 24 |
+
path_in_repo: str
|
| 25 |
+
size_limit: int
|
| 26 |
+
last_modified: float
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class CommitScheduler:
|
| 30 |
+
"""
|
| 31 |
+
Scheduler to upload a local folder to the Hub at regular intervals (e.g. push to hub every 5 minutes).
|
| 32 |
+
|
| 33 |
+
The recommended way to use the scheduler is to use it as a context manager. This ensures that the scheduler is
|
| 34 |
+
properly stopped and the last commit is triggered when the script ends. The scheduler can also be stopped manually
|
| 35 |
+
with the `stop` method. Checkout the [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#scheduled-uploads)
|
| 36 |
+
to learn more about how to use it.
|
| 37 |
+
|
| 38 |
+
Args:
|
| 39 |
+
repo_id (`str`):
|
| 40 |
+
The id of the repo to commit to.
|
| 41 |
+
folder_path (`str` or `Path`):
|
| 42 |
+
Path to the local folder to upload regularly.
|
| 43 |
+
every (`int` or `float`, *optional*):
|
| 44 |
+
The number of minutes between each commit. Defaults to 5 minutes.
|
| 45 |
+
path_in_repo (`str`, *optional*):
|
| 46 |
+
Relative path of the directory in the repo, for example: `"checkpoints/"`. Defaults to the root folder
|
| 47 |
+
of the repository.
|
| 48 |
+
repo_type (`str`, *optional*):
|
| 49 |
+
The type of the repo to commit to. Defaults to `model`.
|
| 50 |
+
revision (`str`, *optional*):
|
| 51 |
+
The revision of the repo to commit to. Defaults to `main`.
|
| 52 |
+
private (`bool`, *optional*):
|
| 53 |
+
Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.
|
| 54 |
+
token (`str`, *optional*):
|
| 55 |
+
The token to use to commit to the repo. Defaults to the token saved on the machine.
|
| 56 |
+
allow_patterns (`List[str]` or `str`, *optional*):
|
| 57 |
+
If provided, only files matching at least one pattern are uploaded.
|
| 58 |
+
ignore_patterns (`List[str]` or `str`, *optional*):
|
| 59 |
+
If provided, files matching any of the patterns are not uploaded.
|
| 60 |
+
squash_history (`bool`, *optional*):
|
| 61 |
+
Whether to squash the history of the repo after each commit. Defaults to `False`. Squashing commits is
|
| 62 |
+
useful to avoid degraded performances on the repo when it grows too large.
|
| 63 |
+
hf_api (`HfApi`, *optional*):
|
| 64 |
+
The [`HfApi`] client to use to commit to the Hub. Can be set with custom settings (user agent, token,...).
|
| 65 |
+
on_before_commit (`Callable[[], None]`, *optional*):
|
| 66 |
+
If specified, a function that will be called before the CommitScheduler lists files to create a commit.
|
| 67 |
+
|
| 68 |
+
Example:
|
| 69 |
+
```py
|
| 70 |
+
>>> from pathlib import Path
|
| 71 |
+
>>> from huggingface_hub import CommitScheduler
|
| 72 |
+
|
| 73 |
+
# Scheduler uploads every 10 minutes
|
| 74 |
+
>>> csv_path = Path("watched_folder/data.csv")
|
| 75 |
+
>>> CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path=csv_path.parent, every=10)
|
| 76 |
+
|
| 77 |
+
>>> with csv_path.open("a") as f:
|
| 78 |
+
... f.write("first line")
|
| 79 |
+
|
| 80 |
+
# Some time later (...)
|
| 81 |
+
>>> with csv_path.open("a") as f:
|
| 82 |
+
... f.write("second line")
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
Example using a context manager:
|
| 86 |
+
```py
|
| 87 |
+
>>> from pathlib import Path
|
| 88 |
+
>>> from huggingface_hub import CommitScheduler
|
| 89 |
+
|
| 90 |
+
>>> with CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path="watched_folder", every=10) as scheduler:
|
| 91 |
+
... csv_path = Path("watched_folder/data.csv")
|
| 92 |
+
... with csv_path.open("a") as f:
|
| 93 |
+
... f.write("first line")
|
| 94 |
+
... (...)
|
| 95 |
+
... with csv_path.open("a") as f:
|
| 96 |
+
... f.write("second line")
|
| 97 |
+
|
| 98 |
+
# Scheduler is now stopped and last commit have been triggered
|
| 99 |
+
```
|
| 100 |
+
"""
|
| 101 |
+
|
| 102 |
+
def __init__(
|
| 103 |
+
self,
|
| 104 |
+
*,
|
| 105 |
+
repo_id: str,
|
| 106 |
+
folder_path: Union[str, Path],
|
| 107 |
+
every: Union[int, float] = 5,
|
| 108 |
+
path_in_repo: str | None = None,
|
| 109 |
+
repo_type: str | None = None,
|
| 110 |
+
revision: str | None = None,
|
| 111 |
+
private: bool | None = None,
|
| 112 |
+
token: str | None = None,
|
| 113 |
+
allow_patterns: list[str] | str | None = None,
|
| 114 |
+
ignore_patterns: list[str] | str | None = None,
|
| 115 |
+
squash_history: bool = False,
|
| 116 |
+
hf_api: HfApi | None = None,
|
| 117 |
+
on_before_commit: Callable[[], None] | None = None,
|
| 118 |
+
) -> None:
|
| 119 |
+
self.api = hf_api or HfApi(token=token)
|
| 120 |
+
self.on_before_commit = on_before_commit
|
| 121 |
+
|
| 122 |
+
# Folder
|
| 123 |
+
self.folder_path = Path(folder_path).expanduser().resolve()
|
| 124 |
+
self.path_in_repo = path_in_repo or ""
|
| 125 |
+
self.allow_patterns = allow_patterns
|
| 126 |
+
|
| 127 |
+
if ignore_patterns is None:
|
| 128 |
+
ignore_patterns = []
|
| 129 |
+
elif isinstance(ignore_patterns, str):
|
| 130 |
+
ignore_patterns = [ignore_patterns]
|
| 131 |
+
self.ignore_patterns = ignore_patterns + DEFAULT_IGNORE_PATTERNS
|
| 132 |
+
|
| 133 |
+
if self.folder_path.is_file():
|
| 134 |
+
raise ValueError(
|
| 135 |
+
f"'folder_path' must be a directory, not a file: '{self.folder_path}'."
|
| 136 |
+
)
|
| 137 |
+
self.folder_path.mkdir(parents=True, exist_ok=True)
|
| 138 |
+
|
| 139 |
+
# Repository
|
| 140 |
+
repo_url = self.api.create_repo(
|
| 141 |
+
repo_id=repo_id, private=private, repo_type=repo_type, exist_ok=True
|
| 142 |
+
)
|
| 143 |
+
self.repo_id = repo_url.repo_id
|
| 144 |
+
self.repo_type = repo_type
|
| 145 |
+
self.revision = revision
|
| 146 |
+
self.token = token
|
| 147 |
+
|
| 148 |
+
self.last_uploaded: Dict[Path, float] = {}
|
| 149 |
+
self.last_push_time: float | None = None
|
| 150 |
+
|
| 151 |
+
if not every > 0:
|
| 152 |
+
raise ValueError(f"'every' must be a positive integer, not '{every}'.")
|
| 153 |
+
self.lock = Lock()
|
| 154 |
+
self.every = every
|
| 155 |
+
self.squash_history = squash_history
|
| 156 |
+
|
| 157 |
+
logger.info(
|
| 158 |
+
f"Scheduled job to push '{self.folder_path}' to '{self.repo_id}' every {self.every} minutes."
|
| 159 |
+
)
|
| 160 |
+
self._scheduler_thread = Thread(target=self._run_scheduler, daemon=True)
|
| 161 |
+
self._scheduler_thread.start()
|
| 162 |
+
atexit.register(self._push_to_hub)
|
| 163 |
+
|
| 164 |
+
self.__stopped = False
|
| 165 |
+
|
| 166 |
+
def stop(self) -> None:
|
| 167 |
+
"""Stop the scheduler.
|
| 168 |
+
|
| 169 |
+
A stopped scheduler cannot be restarted. Mostly for tests purposes.
|
| 170 |
+
"""
|
| 171 |
+
self.__stopped = True
|
| 172 |
+
|
| 173 |
+
def __enter__(self) -> "CommitScheduler":
|
| 174 |
+
return self
|
| 175 |
+
|
| 176 |
+
def __exit__(self, exc_type, exc_value, traceback) -> None:
|
| 177 |
+
# Upload last changes before exiting
|
| 178 |
+
self.trigger().result()
|
| 179 |
+
self.stop()
|
| 180 |
+
return
|
| 181 |
+
|
| 182 |
+
def _run_scheduler(self) -> None:
|
| 183 |
+
"""Dumb thread waiting between each scheduled push to Hub."""
|
| 184 |
+
while True:
|
| 185 |
+
self.last_future = self.trigger()
|
| 186 |
+
time.sleep(self.every * 60)
|
| 187 |
+
if self.__stopped:
|
| 188 |
+
break
|
| 189 |
+
|
| 190 |
+
def trigger(self) -> Future:
|
| 191 |
+
"""Trigger a `push_to_hub` and return a future.
|
| 192 |
+
|
| 193 |
+
This method is automatically called every `every` minutes. You can also call it manually to trigger a commit
|
| 194 |
+
immediately, without waiting for the next scheduled commit.
|
| 195 |
+
"""
|
| 196 |
+
return self.api.run_as_future(self._push_to_hub)
|
| 197 |
+
|
| 198 |
+
def _push_to_hub(self) -> CommitInfo | None:
|
| 199 |
+
if self.__stopped: # If stopped, already scheduled commits are ignored
|
| 200 |
+
return None
|
| 201 |
+
|
| 202 |
+
logger.info("(Background) scheduled commit triggered.")
|
| 203 |
+
try:
|
| 204 |
+
value = self.push_to_hub()
|
| 205 |
+
if self.squash_history:
|
| 206 |
+
logger.info("(Background) squashing repo history.")
|
| 207 |
+
self.api.super_squash_history(
|
| 208 |
+
repo_id=self.repo_id, repo_type=self.repo_type, branch=self.revision
|
| 209 |
+
)
|
| 210 |
+
return value
|
| 211 |
+
except Exception as e:
|
| 212 |
+
logger.error(
|
| 213 |
+
f"Error while pushing to Hub: {e}"
|
| 214 |
+
) # Depending on the setup, error might be silenced
|
| 215 |
+
raise
|
| 216 |
+
|
| 217 |
+
def push_to_hub(self) -> CommitInfo | None:
|
| 218 |
+
"""
|
| 219 |
+
Push folder to the Hub and return the commit info.
|
| 220 |
+
|
| 221 |
+
<Tip warning={true}>
|
| 222 |
+
|
| 223 |
+
This method is not meant to be called directly. It is run in the background by the scheduler, respecting a
|
| 224 |
+
queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency
|
| 225 |
+
issues.
|
| 226 |
+
|
| 227 |
+
</Tip>
|
| 228 |
+
|
| 229 |
+
The default behavior of `push_to_hub` is to assume an append-only folder. It lists all files in the folder and
|
| 230 |
+
uploads only changed files. If no changes are found, the method returns without committing anything. If you want
|
| 231 |
+
to change this behavior, you can inherit from [`CommitScheduler`] and override this method. This can be useful
|
| 232 |
+
for example to compress data together in a single file before committing. For more details and examples, check
|
| 233 |
+
out our [integration guide](https://huggingface.co/docs/huggingface_hub/main/en/guides/upload#scheduled-uploads).
|
| 234 |
+
"""
|
| 235 |
+
# Check files to upload (with lock)
|
| 236 |
+
with self.lock:
|
| 237 |
+
if self.on_before_commit is not None:
|
| 238 |
+
self.on_before_commit()
|
| 239 |
+
|
| 240 |
+
logger.debug("Listing files to upload for scheduled commit.")
|
| 241 |
+
|
| 242 |
+
# List files from folder (taken from `_prepare_upload_folder_additions`)
|
| 243 |
+
relpath_to_abspath = {
|
| 244 |
+
path.relative_to(self.folder_path).as_posix(): path
|
| 245 |
+
for path in sorted(
|
| 246 |
+
self.folder_path.glob("**/*")
|
| 247 |
+
) # sorted to be deterministic
|
| 248 |
+
if path.is_file()
|
| 249 |
+
}
|
| 250 |
+
prefix = f"{self.path_in_repo.strip('/')}/" if self.path_in_repo else ""
|
| 251 |
+
|
| 252 |
+
# Filter with pattern + filter out unchanged files + retrieve current file size
|
| 253 |
+
files_to_upload: List[_FileToUpload] = []
|
| 254 |
+
for relpath in filter_repo_objects(
|
| 255 |
+
relpath_to_abspath.keys(),
|
| 256 |
+
allow_patterns=self.allow_patterns,
|
| 257 |
+
ignore_patterns=self.ignore_patterns,
|
| 258 |
+
):
|
| 259 |
+
local_path = relpath_to_abspath[relpath]
|
| 260 |
+
stat = local_path.stat()
|
| 261 |
+
if (
|
| 262 |
+
self.last_uploaded.get(local_path) is None
|
| 263 |
+
or self.last_uploaded[local_path] != stat.st_mtime
|
| 264 |
+
):
|
| 265 |
+
files_to_upload.append(
|
| 266 |
+
_FileToUpload(
|
| 267 |
+
local_path=local_path,
|
| 268 |
+
path_in_repo=prefix + relpath,
|
| 269 |
+
size_limit=stat.st_size,
|
| 270 |
+
last_modified=stat.st_mtime,
|
| 271 |
+
)
|
| 272 |
+
)
|
| 273 |
+
|
| 274 |
+
# Return if nothing to upload
|
| 275 |
+
if len(files_to_upload) == 0:
|
| 276 |
+
logger.debug("Dropping schedule commit: no changed file to upload.")
|
| 277 |
+
return None
|
| 278 |
+
|
| 279 |
+
# Convert `_FileToUpload` as `CommitOperationAdd` (=> compute file shas + limit to file size)
|
| 280 |
+
logger.debug("Removing unchanged files since previous scheduled commit.")
|
| 281 |
+
add_operations = [
|
| 282 |
+
CommitOperationAdd(
|
| 283 |
+
# TODO: Cap the file to its current size, even if the user append data to it while a scheduled commit is happening
|
| 284 |
+
# (requires an upstream fix for XET-535: `hf_xet` should support `BinaryIO` for upload)
|
| 285 |
+
path_or_fileobj=file_to_upload.local_path,
|
| 286 |
+
path_in_repo=file_to_upload.path_in_repo,
|
| 287 |
+
)
|
| 288 |
+
for file_to_upload in files_to_upload
|
| 289 |
+
]
|
| 290 |
+
|
| 291 |
+
# Upload files (append mode expected - no need for lock)
|
| 292 |
+
logger.debug("Uploading files for scheduled commit.")
|
| 293 |
+
commit_info = self.api.create_commit(
|
| 294 |
+
repo_id=self.repo_id,
|
| 295 |
+
repo_type=self.repo_type,
|
| 296 |
+
operations=add_operations,
|
| 297 |
+
commit_message="Scheduled Commit",
|
| 298 |
+
revision=self.revision,
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
for file in files_to_upload:
|
| 302 |
+
self.last_uploaded[file.local_path] = file.last_modified
|
| 303 |
+
|
| 304 |
+
self.last_push_time = time.time()
|
| 305 |
+
|
| 306 |
+
return commit_info
|
trackio/compression.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import gzip
|
| 4 |
+
import io
|
| 5 |
+
from typing import NoReturn
|
| 6 |
+
|
| 7 |
+
from starlette.datastructures import Headers, MutableHeaders
|
| 8 |
+
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
| 9 |
+
|
| 10 |
+
try:
|
| 11 |
+
import brotli
|
| 12 |
+
|
| 13 |
+
HAS_BROTLI = True
|
| 14 |
+
except ImportError:
|
| 15 |
+
brotli = None
|
| 16 |
+
HAS_BROTLI = False
|
| 17 |
+
|
| 18 |
+
COMPRESSIBLE_CONTENT_TYPES = (
|
| 19 |
+
"application/json",
|
| 20 |
+
"application/javascript",
|
| 21 |
+
"application/manifest+json",
|
| 22 |
+
"application/xml",
|
| 23 |
+
"application/xhtml+xml",
|
| 24 |
+
"image/svg+xml",
|
| 25 |
+
"text/",
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class CompressionMiddleware:
|
| 30 |
+
"""Negotiates Brotli, then gzip, then identity based on Accept-Encoding.
|
| 31 |
+
|
| 32 |
+
Only text-like content types are compressed, so already-compressed media
|
| 33 |
+
(images, audio, video) and zero-copy file sends pass through untouched.
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
def __init__(
|
| 37 |
+
self,
|
| 38 |
+
app: ASGIApp,
|
| 39 |
+
minimum_size: int = 512,
|
| 40 |
+
gzip_level: int = 6,
|
| 41 |
+
brotli_quality: int = 5,
|
| 42 |
+
) -> None:
|
| 43 |
+
self.app = app
|
| 44 |
+
self.minimum_size = minimum_size
|
| 45 |
+
self.gzip_level = gzip_level
|
| 46 |
+
self.brotli_quality = brotli_quality
|
| 47 |
+
|
| 48 |
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
| 49 |
+
if scope["type"] != "http":
|
| 50 |
+
await self.app(scope, receive, send)
|
| 51 |
+
return
|
| 52 |
+
|
| 53 |
+
accept_encoding = Headers(scope=scope).get("Accept-Encoding", "")
|
| 54 |
+
responder: _CompressionResponder
|
| 55 |
+
if HAS_BROTLI and "br" in accept_encoding:
|
| 56 |
+
responder = _BrotliResponder(
|
| 57 |
+
self.app, self.minimum_size, self.brotli_quality
|
| 58 |
+
)
|
| 59 |
+
elif "gzip" in accept_encoding:
|
| 60 |
+
responder = _GZipResponder(self.app, self.minimum_size, self.gzip_level)
|
| 61 |
+
else:
|
| 62 |
+
responder = _CompressionResponder(self.app, self.minimum_size)
|
| 63 |
+
|
| 64 |
+
await responder(scope, receive, send)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class _CompressionResponder:
|
| 68 |
+
content_encoding: str = ""
|
| 69 |
+
|
| 70 |
+
def __init__(self, app: ASGIApp, minimum_size: int) -> None:
|
| 71 |
+
self.app = app
|
| 72 |
+
self.minimum_size = minimum_size
|
| 73 |
+
self.send: Send = _unattached_send
|
| 74 |
+
self.initial_message: Message = {}
|
| 75 |
+
self.started = False
|
| 76 |
+
self.passthrough = True
|
| 77 |
+
|
| 78 |
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
| 79 |
+
self.send = send
|
| 80 |
+
await self.app(scope, receive, self.send_with_compression)
|
| 81 |
+
|
| 82 |
+
async def send_with_compression(self, message: Message) -> None:
|
| 83 |
+
message_type = message["type"]
|
| 84 |
+
if message_type == "http.response.start":
|
| 85 |
+
self.initial_message = message
|
| 86 |
+
headers = Headers(raw=message["headers"])
|
| 87 |
+
content_type = headers.get("content-type", "")
|
| 88 |
+
already_encoded = "content-encoding" in headers
|
| 89 |
+
self.passthrough = already_encoded or not content_type.startswith(
|
| 90 |
+
COMPRESSIBLE_CONTENT_TYPES
|
| 91 |
+
)
|
| 92 |
+
elif message_type == "http.response.body" and self.passthrough:
|
| 93 |
+
if not self.started:
|
| 94 |
+
self.started = True
|
| 95 |
+
await self.send(self.initial_message)
|
| 96 |
+
await self.send(message)
|
| 97 |
+
elif message_type == "http.response.body" and not self.started:
|
| 98 |
+
self.started = True
|
| 99 |
+
body = message.get("body", b"")
|
| 100 |
+
more_body = message.get("more_body", False)
|
| 101 |
+
if len(body) < self.minimum_size and not more_body:
|
| 102 |
+
await self.send(self.initial_message)
|
| 103 |
+
await self.send(message)
|
| 104 |
+
else:
|
| 105 |
+
compressed = self.apply_compression(body, more_body=more_body)
|
| 106 |
+
headers = MutableHeaders(raw=self.initial_message["headers"])
|
| 107 |
+
headers.add_vary_header("Accept-Encoding")
|
| 108 |
+
if compressed != body:
|
| 109 |
+
headers["Content-Encoding"] = self.content_encoding
|
| 110 |
+
if more_body:
|
| 111 |
+
del headers["Content-Length"]
|
| 112 |
+
else:
|
| 113 |
+
headers["Content-Length"] = str(len(compressed))
|
| 114 |
+
message["body"] = compressed
|
| 115 |
+
await self.send(self.initial_message)
|
| 116 |
+
await self.send(message)
|
| 117 |
+
elif message_type == "http.response.body":
|
| 118 |
+
body = message.get("body", b"")
|
| 119 |
+
more_body = message.get("more_body", False)
|
| 120 |
+
message["body"] = self.apply_compression(body, more_body=more_body)
|
| 121 |
+
await self.send(message)
|
| 122 |
+
elif message_type == "http.response.pathsend":
|
| 123 |
+
await self.send(self.initial_message)
|
| 124 |
+
await self.send(message)
|
| 125 |
+
|
| 126 |
+
def apply_compression(self, body: bytes, *, more_body: bool) -> bytes:
|
| 127 |
+
return body
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
class _GZipResponder(_CompressionResponder):
|
| 131 |
+
content_encoding = "gzip"
|
| 132 |
+
|
| 133 |
+
def __init__(self, app: ASGIApp, minimum_size: int, compresslevel: int) -> None:
|
| 134 |
+
super().__init__(app, minimum_size)
|
| 135 |
+
self.buffer = io.BytesIO()
|
| 136 |
+
self.file = gzip.GzipFile(
|
| 137 |
+
mode="wb", fileobj=self.buffer, compresslevel=compresslevel
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
| 141 |
+
with self.buffer, self.file:
|
| 142 |
+
await super().__call__(scope, receive, send)
|
| 143 |
+
|
| 144 |
+
def apply_compression(self, body: bytes, *, more_body: bool) -> bytes:
|
| 145 |
+
self.file.write(body)
|
| 146 |
+
if not more_body:
|
| 147 |
+
self.file.close()
|
| 148 |
+
data = self.buffer.getvalue()
|
| 149 |
+
self.buffer.seek(0)
|
| 150 |
+
self.buffer.truncate()
|
| 151 |
+
return data
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
class _BrotliResponder(_CompressionResponder):
|
| 155 |
+
content_encoding = "br"
|
| 156 |
+
|
| 157 |
+
def __init__(self, app: ASGIApp, minimum_size: int, quality: int) -> None:
|
| 158 |
+
super().__init__(app, minimum_size)
|
| 159 |
+
self.compressor = brotli.Compressor(quality=quality)
|
| 160 |
+
|
| 161 |
+
def apply_compression(self, body: bytes, *, more_body: bool) -> bytes:
|
| 162 |
+
data = self.compressor.process(body)
|
| 163 |
+
if not more_body:
|
| 164 |
+
data += self.compressor.finish()
|
| 165 |
+
return data
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
async def _unattached_send(message: Message) -> NoReturn:
|
| 169 |
+
raise RuntimeError("send awaitable not set")
|
trackio/context_vars.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import contextvars
|
| 2 |
+
from typing import TYPE_CHECKING
|
| 3 |
+
|
| 4 |
+
if TYPE_CHECKING:
|
| 5 |
+
from trackio.run import Run
|
| 6 |
+
|
| 7 |
+
current_run: contextvars.ContextVar["Run | None"] = contextvars.ContextVar(
|
| 8 |
+
"current_run", default=None
|
| 9 |
+
)
|
| 10 |
+
current_project: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
| 11 |
+
"current_project", default=None
|
| 12 |
+
)
|
| 13 |
+
current_server: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
| 14 |
+
"current_server", default=None
|
| 15 |
+
)
|
| 16 |
+
current_space_id: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
| 17 |
+
"current_space_id", default=None
|
| 18 |
+
)
|
| 19 |
+
current_server_write_token: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
| 20 |
+
"current_server_write_token", default=None
|
| 21 |
+
)
|
trackio/cpu.py
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib
|
| 2 |
+
import threading
|
| 3 |
+
import time
|
| 4 |
+
import warnings
|
| 5 |
+
from typing import TYPE_CHECKING, Any
|
| 6 |
+
|
| 7 |
+
from trackio import context_vars
|
| 8 |
+
|
| 9 |
+
if TYPE_CHECKING:
|
| 10 |
+
from trackio.run import Run
|
| 11 |
+
|
| 12 |
+
psutil: Any = None
|
| 13 |
+
PSUTIL_AVAILABLE = False
|
| 14 |
+
_psutil_lock = threading.Lock()
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _ensure_psutil():
|
| 18 |
+
global PSUTIL_AVAILABLE, psutil
|
| 19 |
+
if PSUTIL_AVAILABLE:
|
| 20 |
+
return psutil
|
| 21 |
+
with _psutil_lock:
|
| 22 |
+
if PSUTIL_AVAILABLE:
|
| 23 |
+
return psutil
|
| 24 |
+
try:
|
| 25 |
+
psutil = importlib.import_module("psutil")
|
| 26 |
+
PSUTIL_AVAILABLE = True
|
| 27 |
+
return psutil
|
| 28 |
+
except ImportError:
|
| 29 |
+
raise ImportError(
|
| 30 |
+
"psutil is required for CPU and RAM monitoring. "
|
| 31 |
+
"Install it with: pip install psutil"
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def cpu_available() -> bool:
|
| 36 |
+
"""
|
| 37 |
+
Check if CPU and RAM monitoring is available.
|
| 38 |
+
|
| 39 |
+
Returns True if psutil is installed.
|
| 40 |
+
"""
|
| 41 |
+
try:
|
| 42 |
+
_ensure_psutil()
|
| 43 |
+
return True
|
| 44 |
+
except ImportError:
|
| 45 |
+
return False
|
| 46 |
+
except Exception:
|
| 47 |
+
return False
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def collect_cpu_metrics(
|
| 51 |
+
prev_disk_counters: Any = None,
|
| 52 |
+
prev_net_counters: Any = None,
|
| 53 |
+
elapsed: float | None = None,
|
| 54 |
+
include_static: bool = True,
|
| 55 |
+
) -> dict:
|
| 56 |
+
"""
|
| 57 |
+
Collect CPU, RAM, disk, network, and sensor metrics using psutil.
|
| 58 |
+
|
| 59 |
+
Args:
|
| 60 |
+
prev_disk_counters: Previous disk I/O counters for computing read/write rates.
|
| 61 |
+
If None, only cumulative values are reported.
|
| 62 |
+
prev_net_counters: Previous network I/O counters for computing send/recv rates.
|
| 63 |
+
If None, only cumulative values are reported.
|
| 64 |
+
elapsed: Seconds since prev_disk_counters and prev_net_counters were captured.
|
| 65 |
+
If None or non-positive, cumulative values are reported instead of rates.
|
| 66 |
+
include_static: Whether to include mostly static metrics such as CPU
|
| 67 |
+
frequency and core counts.
|
| 68 |
+
|
| 69 |
+
Returns:
|
| 70 |
+
Dictionary of system metrics.
|
| 71 |
+
"""
|
| 72 |
+
if not PSUTIL_AVAILABLE:
|
| 73 |
+
try:
|
| 74 |
+
_ensure_psutil()
|
| 75 |
+
except ImportError:
|
| 76 |
+
return {}
|
| 77 |
+
|
| 78 |
+
metrics = {}
|
| 79 |
+
|
| 80 |
+
try:
|
| 81 |
+
per_core = psutil.cpu_percent(interval=0.1, percpu=True)
|
| 82 |
+
for i, pct in enumerate(per_core):
|
| 83 |
+
metrics[f"cpu/{i}/utilization"] = pct
|
| 84 |
+
if per_core:
|
| 85 |
+
metrics["cpu/utilization"] = sum(per_core) / len(per_core)
|
| 86 |
+
except Exception:
|
| 87 |
+
pass
|
| 88 |
+
|
| 89 |
+
if include_static:
|
| 90 |
+
try:
|
| 91 |
+
cpu_freq = psutil.cpu_freq()
|
| 92 |
+
if cpu_freq:
|
| 93 |
+
metrics["cpu/frequency"] = cpu_freq.current
|
| 94 |
+
if cpu_freq.max > 0:
|
| 95 |
+
metrics["cpu/frequency_max"] = cpu_freq.max
|
| 96 |
+
except Exception:
|
| 97 |
+
pass
|
| 98 |
+
|
| 99 |
+
try:
|
| 100 |
+
cpu_count_logical = psutil.cpu_count(logical=True)
|
| 101 |
+
if cpu_count_logical is not None:
|
| 102 |
+
metrics["cpu/count_logical"] = cpu_count_logical
|
| 103 |
+
cpu_count_physical = psutil.cpu_count(logical=False)
|
| 104 |
+
if cpu_count_physical is not None:
|
| 105 |
+
metrics["cpu/count_physical"] = cpu_count_physical
|
| 106 |
+
except Exception:
|
| 107 |
+
pass
|
| 108 |
+
|
| 109 |
+
try:
|
| 110 |
+
mem = psutil.virtual_memory()
|
| 111 |
+
metrics["memory/used"] = mem.used / (1024**3)
|
| 112 |
+
metrics["memory/total"] = mem.total / (1024**3)
|
| 113 |
+
metrics["memory/available"] = mem.available / (1024**3)
|
| 114 |
+
metrics["memory/percent"] = mem.percent
|
| 115 |
+
except Exception:
|
| 116 |
+
pass
|
| 117 |
+
|
| 118 |
+
try:
|
| 119 |
+
swap = psutil.swap_memory()
|
| 120 |
+
metrics["swap/used"] = swap.used / (1024**3)
|
| 121 |
+
metrics["swap/total"] = swap.total / (1024**3)
|
| 122 |
+
metrics["swap/percent"] = swap.percent
|
| 123 |
+
except Exception:
|
| 124 |
+
pass
|
| 125 |
+
|
| 126 |
+
try:
|
| 127 |
+
disk = psutil.disk_io_counters()
|
| 128 |
+
if disk is not None:
|
| 129 |
+
if prev_disk_counters is not None and elapsed and elapsed > 0:
|
| 130 |
+
metrics["disk/read_mb_per_sec"] = (
|
| 131 |
+
(disk.read_bytes - prev_disk_counters.read_bytes)
|
| 132 |
+
/ elapsed
|
| 133 |
+
/ (1024**2)
|
| 134 |
+
)
|
| 135 |
+
metrics["disk/write_mb_per_sec"] = (
|
| 136 |
+
(disk.write_bytes - prev_disk_counters.write_bytes)
|
| 137 |
+
/ elapsed
|
| 138 |
+
/ (1024**2)
|
| 139 |
+
)
|
| 140 |
+
metrics["disk/read_iops"] = (
|
| 141 |
+
disk.read_count - prev_disk_counters.read_count
|
| 142 |
+
) / elapsed
|
| 143 |
+
metrics["disk/write_iops"] = (
|
| 144 |
+
disk.write_count - prev_disk_counters.write_count
|
| 145 |
+
) / elapsed
|
| 146 |
+
else:
|
| 147 |
+
metrics["disk/read_bytes"] = disk.read_bytes
|
| 148 |
+
metrics["disk/write_bytes"] = disk.write_bytes
|
| 149 |
+
except Exception:
|
| 150 |
+
pass
|
| 151 |
+
|
| 152 |
+
try:
|
| 153 |
+
net = psutil.net_io_counters()
|
| 154 |
+
if net is not None:
|
| 155 |
+
if prev_net_counters is not None and elapsed and elapsed > 0:
|
| 156 |
+
metrics["network/sent_mb_per_sec"] = (
|
| 157 |
+
(net.bytes_sent - prev_net_counters.bytes_sent)
|
| 158 |
+
/ elapsed
|
| 159 |
+
/ (1024**2)
|
| 160 |
+
)
|
| 161 |
+
metrics["network/recv_mb_per_sec"] = (
|
| 162 |
+
(net.bytes_recv - prev_net_counters.bytes_recv)
|
| 163 |
+
/ elapsed
|
| 164 |
+
/ (1024**2)
|
| 165 |
+
)
|
| 166 |
+
else:
|
| 167 |
+
metrics["network/sent_bytes"] = net.bytes_sent
|
| 168 |
+
metrics["network/recv_bytes"] = net.bytes_recv
|
| 169 |
+
except Exception:
|
| 170 |
+
pass
|
| 171 |
+
|
| 172 |
+
try:
|
| 173 |
+
sensors = psutil.sensors_temperatures()
|
| 174 |
+
if sensors:
|
| 175 |
+
for chip_name, entries in sensors.items():
|
| 176 |
+
for i, entry in enumerate(entries):
|
| 177 |
+
label = (
|
| 178 |
+
entry.label.strip()
|
| 179 |
+
if entry.label and entry.label.strip()
|
| 180 |
+
else f"{chip_name}_{i}"
|
| 181 |
+
)
|
| 182 |
+
metrics[f"temp/{label}"] = entry.current
|
| 183 |
+
except Exception:
|
| 184 |
+
pass
|
| 185 |
+
|
| 186 |
+
try:
|
| 187 |
+
battery = psutil.sensors_battery()
|
| 188 |
+
if battery is not None:
|
| 189 |
+
metrics["battery/percent"] = battery.percent
|
| 190 |
+
metrics["battery/power_plugged"] = int(battery.power_plugged)
|
| 191 |
+
except Exception:
|
| 192 |
+
pass
|
| 193 |
+
|
| 194 |
+
return metrics
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
class CpuMonitor:
|
| 198 |
+
def __init__(self, run: "Run", interval: float = 10.0):
|
| 199 |
+
self._run = run
|
| 200 |
+
self._interval = interval
|
| 201 |
+
self._stop_flag = threading.Event()
|
| 202 |
+
self._thread: "threading.Thread | None" = None
|
| 203 |
+
self._last_disk_counters: Any = None
|
| 204 |
+
self._last_net_counters: Any = None
|
| 205 |
+
self._last_time: float | None = None
|
| 206 |
+
self._include_static_next = True
|
| 207 |
+
|
| 208 |
+
def start(self):
|
| 209 |
+
if not PSUTIL_AVAILABLE:
|
| 210 |
+
try:
|
| 211 |
+
_ensure_psutil()
|
| 212 |
+
except ImportError:
|
| 213 |
+
warnings.warn(
|
| 214 |
+
"auto_log_cpu=True but psutil is not installed. "
|
| 215 |
+
"CPU and RAM logging disabled. Install with: pip install psutil"
|
| 216 |
+
)
|
| 217 |
+
return
|
| 218 |
+
|
| 219 |
+
self._thread = threading.Thread(target=self._monitor_loop, daemon=True)
|
| 220 |
+
self._thread.start()
|
| 221 |
+
|
| 222 |
+
def stop(self):
|
| 223 |
+
self._stop_flag.set()
|
| 224 |
+
if self._thread is not None:
|
| 225 |
+
self._thread.join(timeout=2.0)
|
| 226 |
+
|
| 227 |
+
def _monitor_loop(self):
|
| 228 |
+
try:
|
| 229 |
+
self._last_disk_counters = psutil.disk_io_counters()
|
| 230 |
+
except Exception:
|
| 231 |
+
pass
|
| 232 |
+
try:
|
| 233 |
+
self._last_net_counters = psutil.net_io_counters()
|
| 234 |
+
except Exception:
|
| 235 |
+
pass
|
| 236 |
+
self._last_time = time.monotonic()
|
| 237 |
+
|
| 238 |
+
while not self._stop_flag.is_set():
|
| 239 |
+
self._stop_flag.wait(timeout=self._interval)
|
| 240 |
+
if self._stop_flag.is_set():
|
| 241 |
+
break
|
| 242 |
+
try:
|
| 243 |
+
now = time.monotonic()
|
| 244 |
+
elapsed = now - self._last_time if self._last_time is not None else None
|
| 245 |
+
metrics = collect_cpu_metrics(
|
| 246 |
+
prev_disk_counters=self._last_disk_counters,
|
| 247 |
+
prev_net_counters=self._last_net_counters,
|
| 248 |
+
elapsed=elapsed,
|
| 249 |
+
include_static=self._include_static_next,
|
| 250 |
+
)
|
| 251 |
+
self._include_static_next = False
|
| 252 |
+
try:
|
| 253 |
+
self._last_disk_counters = psutil.disk_io_counters()
|
| 254 |
+
except Exception:
|
| 255 |
+
self._last_disk_counters = None
|
| 256 |
+
try:
|
| 257 |
+
self._last_net_counters = psutil.net_io_counters()
|
| 258 |
+
except Exception:
|
| 259 |
+
self._last_net_counters = None
|
| 260 |
+
self._last_time = now
|
| 261 |
+
if metrics:
|
| 262 |
+
self._run.log_system(metrics)
|
| 263 |
+
except Exception:
|
| 264 |
+
pass
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def log_cpu(run: "Run | None" = None) -> dict:
|
| 268 |
+
"""
|
| 269 |
+
Log CPU, RAM, disk, network, and sensor metrics to the current or specified run
|
| 270 |
+
as system metrics.
|
| 271 |
+
|
| 272 |
+
Args:
|
| 273 |
+
run: Optional Run instance. If None, uses current run from context.
|
| 274 |
+
|
| 275 |
+
Returns:
|
| 276 |
+
dict: The system metrics that were logged.
|
| 277 |
+
|
| 278 |
+
Example:
|
| 279 |
+
```python
|
| 280 |
+
import trackio
|
| 281 |
+
|
| 282 |
+
run = trackio.init(project="my-project")
|
| 283 |
+
trackio.log({"loss": 0.5})
|
| 284 |
+
trackio.log_cpu()
|
| 285 |
+
```
|
| 286 |
+
"""
|
| 287 |
+
if run is None:
|
| 288 |
+
run = context_vars.current_run.get()
|
| 289 |
+
if run is None:
|
| 290 |
+
raise RuntimeError("Call trackio.init() before trackio.log_cpu().")
|
| 291 |
+
|
| 292 |
+
try:
|
| 293 |
+
_ensure_psutil()
|
| 294 |
+
except ImportError:
|
| 295 |
+
warnings.warn(
|
| 296 |
+
"trackio.log_cpu() requires psutil. Install it with: pip install trackio[cpu]"
|
| 297 |
+
)
|
| 298 |
+
return {}
|
| 299 |
+
|
| 300 |
+
metrics = collect_cpu_metrics()
|
| 301 |
+
if metrics:
|
| 302 |
+
run.log_system(metrics)
|
| 303 |
+
return metrics
|
trackio/deploy.py
ADDED
|
@@ -0,0 +1,1332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib.metadata
|
| 2 |
+
import io
|
| 3 |
+
import json as json_mod
|
| 4 |
+
import os
|
| 5 |
+
import tempfile
|
| 6 |
+
import threading
|
| 7 |
+
import time
|
| 8 |
+
import warnings
|
| 9 |
+
from collections import Counter
|
| 10 |
+
from importlib.resources import files
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
import httpx
|
| 14 |
+
import huggingface_hub
|
| 15 |
+
from gradio_client import handle_file
|
| 16 |
+
from httpx import ReadTimeout
|
| 17 |
+
from huggingface_hub import Volume
|
| 18 |
+
from huggingface_hub.errors import (
|
| 19 |
+
BucketNotFoundError,
|
| 20 |
+
HfHubHTTPError,
|
| 21 |
+
RepositoryNotFoundError,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
import trackio
|
| 25 |
+
from trackio.bucket_storage import (
|
| 26 |
+
create_bucket_if_not_exists,
|
| 27 |
+
export_from_bucket_for_static,
|
| 28 |
+
upload_project_to_bucket,
|
| 29 |
+
upload_project_to_bucket_for_static,
|
| 30 |
+
)
|
| 31 |
+
from trackio.cas import PARTIAL_BLOB_GLOB
|
| 32 |
+
from trackio.frontend_config import resolve_frontend_dir
|
| 33 |
+
from trackio.pending_uploads import replay_pending_uploads
|
| 34 |
+
from trackio.remote_client import RemoteClient
|
| 35 |
+
from trackio.sqlite_storage import SQLiteStorage
|
| 36 |
+
from trackio.utils import (
|
| 37 |
+
get_or_create_project_hash,
|
| 38 |
+
on_spaces,
|
| 39 |
+
preprocess_space_and_dataset_ids,
|
| 40 |
+
project_artifacts_dir,
|
| 41 |
+
project_media_dir,
|
| 42 |
+
warn_dataset_persistence_deprecated,
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
SPACE_HOST_URL = "https://{user_name}-{space_name}.hf.space/"
|
| 46 |
+
SPACE_URL = "https://huggingface.co/spaces/{space_id}"
|
| 47 |
+
_BOLD_ORANGE = "\033[1m\033[38;5;208m"
|
| 48 |
+
_RESET = "\033[0m"
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def raise_if_space_is_frozen_for_logging(space_id: str) -> None:
|
| 52 |
+
try:
|
| 53 |
+
info = huggingface_hub.HfApi().space_info(space_id)
|
| 54 |
+
except RepositoryNotFoundError:
|
| 55 |
+
return
|
| 56 |
+
if getattr(info, "sdk", None) == "static":
|
| 57 |
+
raise RuntimeError(
|
| 58 |
+
f"Cannot log to Hugging Face Space '{space_id}' because it has been frozen "
|
| 59 |
+
f"(it uses the static SDK: a read-only dashboard with no live Trackio server).\n\n"
|
| 60 |
+
f"Use a different space_id for training, or create a new Gradio Trackio Space. "
|
| 61 |
+
f"Freezing converts a live Gradio Space to static after a run; a frozen Space "
|
| 62 |
+
f'cannot accept new logs. See trackio.sync(..., sdk="static") in the Trackio docs.'
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _readme_linked_hub_yaml(dataset_id: str | None) -> str:
|
| 67 |
+
if dataset_id is not None:
|
| 68 |
+
return f"datasets:\n - {dataset_id}\n"
|
| 69 |
+
return ""
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
_CUSTOM_SPACE_FRONTEND_DIR = "trackio_custom_frontend"
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _space_app_py(frontend_dir: str | None = None) -> str:
|
| 76 |
+
if frontend_dir is None:
|
| 77 |
+
return "import trackio\ntrackio.show()\n"
|
| 78 |
+
return f'import trackio\ntrackio.show(frontend_dir="{frontend_dir}")\n'
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _upload_frontend_folder(
|
| 82 |
+
hf_api: huggingface_hub.HfApi,
|
| 83 |
+
*,
|
| 84 |
+
repo_id: str,
|
| 85 |
+
repo_type: str,
|
| 86 |
+
folder_path: str | Path,
|
| 87 |
+
path_in_repo: str | None = None,
|
| 88 |
+
) -> None:
|
| 89 |
+
kwargs = {
|
| 90 |
+
"repo_id": repo_id,
|
| 91 |
+
"repo_type": repo_type,
|
| 92 |
+
"folder_path": str(folder_path),
|
| 93 |
+
}
|
| 94 |
+
if path_in_repo is not None:
|
| 95 |
+
kwargs["path_in_repo"] = path_in_repo
|
| 96 |
+
hf_api.upload_folder(**kwargs)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _retry_hf_write(op_name: str, fn, retries: int = 4, initial_delay: float = 1.5):
|
| 100 |
+
delay = initial_delay
|
| 101 |
+
for attempt in range(1, retries + 1):
|
| 102 |
+
try:
|
| 103 |
+
return fn()
|
| 104 |
+
except ReadTimeout:
|
| 105 |
+
if attempt == retries:
|
| 106 |
+
raise
|
| 107 |
+
print(
|
| 108 |
+
f"* {op_name} timed out (attempt {attempt}/{retries}). Retrying in {delay:.1f}s..."
|
| 109 |
+
)
|
| 110 |
+
time.sleep(delay)
|
| 111 |
+
delay = min(delay * 2, 12)
|
| 112 |
+
except HfHubHTTPError as e:
|
| 113 |
+
status = e.response.status_code if e.response is not None else None
|
| 114 |
+
if status is None or status < 500 or attempt == retries:
|
| 115 |
+
raise
|
| 116 |
+
print(
|
| 117 |
+
f"* {op_name} failed with HTTP {status} (attempt {attempt}/{retries}). Retrying in {delay:.1f}s..."
|
| 118 |
+
)
|
| 119 |
+
time.sleep(delay)
|
| 120 |
+
delay = min(delay * 2, 12)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _get_space_volumes(
|
| 124 |
+
space_id: str, hf_api: huggingface_hub.HfApi | None = None
|
| 125 |
+
) -> list[Volume]:
|
| 126 |
+
"""
|
| 127 |
+
Return mounted volumes for a Space.
|
| 128 |
+
|
| 129 |
+
`HfApi.get_space_runtime()` does not always populate `volumes`, even when the
|
| 130 |
+
mount exists. Fall back to `space_info().runtime.volumes`, which currently
|
| 131 |
+
carries the volume metadata for running Spaces.
|
| 132 |
+
"""
|
| 133 |
+
hf_api = hf_api or huggingface_hub.HfApi()
|
| 134 |
+
runtime = hf_api.get_space_runtime(space_id)
|
| 135 |
+
if runtime.volumes:
|
| 136 |
+
return list(runtime.volumes)
|
| 137 |
+
|
| 138 |
+
info = hf_api.space_info(space_id)
|
| 139 |
+
if info.runtime and info.runtime.volumes:
|
| 140 |
+
return list(info.runtime.volumes)
|
| 141 |
+
|
| 142 |
+
return []
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _get_space_bucket_at_data_mount(
|
| 146 |
+
space_id: str, hf_api: huggingface_hub.HfApi | None = None
|
| 147 |
+
) -> str | None:
|
| 148 |
+
for volume in _get_space_volumes(space_id, hf_api=hf_api):
|
| 149 |
+
if volume.type == "bucket" and volume.mount_path == "/data":
|
| 150 |
+
return volume.source
|
| 151 |
+
return None
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def _get_existing_space_bucket(
|
| 155 |
+
space_id: str, hf_api: huggingface_hub.HfApi | None = None
|
| 156 |
+
) -> str | None:
|
| 157 |
+
"""Return the Trackio bucket for a Space, preferring the canonical /data mount."""
|
| 158 |
+
bucket_at_data = _get_space_bucket_at_data_mount(space_id, hf_api=hf_api)
|
| 159 |
+
if bucket_at_data is not None:
|
| 160 |
+
return bucket_at_data
|
| 161 |
+
|
| 162 |
+
for volume in _get_space_volumes(space_id, hf_api=hf_api):
|
| 163 |
+
if volume.type == "bucket":
|
| 164 |
+
return volume.source
|
| 165 |
+
return None
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def _get_existing_static_space_bucket(
|
| 169 |
+
space_id: str, hf_api: huggingface_hub.HfApi | None = None
|
| 170 |
+
) -> str | None:
|
| 171 |
+
hf_api = hf_api or huggingface_hub.HfApi()
|
| 172 |
+
try:
|
| 173 |
+
config_path = hf_api.hf_hub_download(
|
| 174 |
+
repo_id=space_id,
|
| 175 |
+
repo_type="space",
|
| 176 |
+
filename="config.json",
|
| 177 |
+
)
|
| 178 |
+
except (FileNotFoundError, HfHubHTTPError, OSError, ValueError):
|
| 179 |
+
return None
|
| 180 |
+
|
| 181 |
+
try:
|
| 182 |
+
with open(config_path, encoding="utf-8") as config_file:
|
| 183 |
+
config = json_mod.load(config_file)
|
| 184 |
+
except (OSError, ValueError, TypeError):
|
| 185 |
+
return None
|
| 186 |
+
|
| 187 |
+
bucket_id = config.get("bucket_id")
|
| 188 |
+
if isinstance(bucket_id, str) and bucket_id:
|
| 189 |
+
return bucket_id
|
| 190 |
+
return None
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _ensure_bucket_mounted_at_data(
|
| 194 |
+
space_id: str,
|
| 195 |
+
bucket_id: str,
|
| 196 |
+
hf_api: huggingface_hub.HfApi | None = None,
|
| 197 |
+
) -> None:
|
| 198 |
+
hf_api = hf_api or huggingface_hub.HfApi()
|
| 199 |
+
existing = _get_space_volumes(space_id, hf_api=hf_api)
|
| 200 |
+
already_mounted = any(
|
| 201 |
+
v.type == "bucket" and v.source == bucket_id and v.mount_path == "/data"
|
| 202 |
+
for v in existing
|
| 203 |
+
)
|
| 204 |
+
if not already_mounted:
|
| 205 |
+
preserved = [
|
| 206 |
+
v
|
| 207 |
+
for v in existing
|
| 208 |
+
if not (
|
| 209 |
+
v.type == "bucket"
|
| 210 |
+
and (v.source == bucket_id or v.mount_path == "/data")
|
| 211 |
+
)
|
| 212 |
+
]
|
| 213 |
+
hf_api.set_space_volumes(
|
| 214 |
+
space_id,
|
| 215 |
+
preserved + [Volume(type="bucket", source=bucket_id, mount_path="/data")],
|
| 216 |
+
)
|
| 217 |
+
print(f"* Attached bucket {bucket_id} at '/data'")
|
| 218 |
+
|
| 219 |
+
existing_variables = hf_api.get_space_variables(space_id)
|
| 220 |
+
current_trackio_dir = getattr(existing_variables.get("TRACKIO_DIR"), "value", None)
|
| 221 |
+
if current_trackio_dir != "/data/trackio":
|
| 222 |
+
huggingface_hub.add_space_variable(space_id, "TRACKIO_DIR", "/data/trackio")
|
| 223 |
+
current_bucket_id = getattr(
|
| 224 |
+
existing_variables.get("TRACKIO_BUCKET_ID"), "value", None
|
| 225 |
+
)
|
| 226 |
+
if current_bucket_id != bucket_id:
|
| 227 |
+
huggingface_hub.add_space_variable(space_id, "TRACKIO_BUCKET_ID", bucket_id)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _bucket_exists(bucket_id: str, hf_api: huggingface_hub.HfApi | None = None) -> bool:
|
| 231 |
+
hf_api = hf_api or huggingface_hub.HfApi()
|
| 232 |
+
try:
|
| 233 |
+
hf_api.bucket_info(bucket_id)
|
| 234 |
+
return True
|
| 235 |
+
except BucketNotFoundError:
|
| 236 |
+
return False
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def _find_available_bucket_id(
|
| 240 |
+
preferred_bucket_id: str, hf_api: huggingface_hub.HfApi | None = None
|
| 241 |
+
) -> str:
|
| 242 |
+
hf_api = hf_api or huggingface_hub.HfApi()
|
| 243 |
+
if not _bucket_exists(preferred_bucket_id, hf_api):
|
| 244 |
+
return preferred_bucket_id
|
| 245 |
+
|
| 246 |
+
suffix = 2
|
| 247 |
+
while True:
|
| 248 |
+
candidate = f"{preferred_bucket_id}-{suffix}"
|
| 249 |
+
if not _bucket_exists(candidate, hf_api):
|
| 250 |
+
return candidate
|
| 251 |
+
suffix += 1
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def resolve_auto_bucket_id(
|
| 255 |
+
space_id: str,
|
| 256 |
+
preferred_bucket_id: str,
|
| 257 |
+
hf_api: huggingface_hub.HfApi | None = None,
|
| 258 |
+
) -> str:
|
| 259 |
+
"""
|
| 260 |
+
Resolve the bucket to use for an auto-generated bucket ID.
|
| 261 |
+
|
| 262 |
+
Rules:
|
| 263 |
+
- Existing Space with a bucket mounted at /data -> reuse that bucket.
|
| 264 |
+
- Existing static Space with a bucket_id in config.json -> reuse that bucket.
|
| 265 |
+
- Otherwise -> use the preferred auto bucket ID if free, or a suffixed variant.
|
| 266 |
+
"""
|
| 267 |
+
hf_api = hf_api or huggingface_hub.HfApi()
|
| 268 |
+
try:
|
| 269 |
+
info = hf_api.space_info(space_id)
|
| 270 |
+
except RepositoryNotFoundError:
|
| 271 |
+
pass
|
| 272 |
+
else:
|
| 273 |
+
existing_bucket_id = _get_existing_space_bucket(space_id, hf_api=hf_api)
|
| 274 |
+
if existing_bucket_id is None and getattr(info, "sdk", None) == "static":
|
| 275 |
+
existing_bucket_id = _get_existing_static_space_bucket(
|
| 276 |
+
space_id, hf_api=hf_api
|
| 277 |
+
)
|
| 278 |
+
if existing_bucket_id is not None:
|
| 279 |
+
return existing_bucket_id
|
| 280 |
+
|
| 281 |
+
bucket_id = _find_available_bucket_id(preferred_bucket_id, hf_api)
|
| 282 |
+
if bucket_id != preferred_bucket_id:
|
| 283 |
+
print(
|
| 284 |
+
f"* Auto-generated bucket {preferred_bucket_id} already exists; "
|
| 285 |
+
f"using {bucket_id} instead"
|
| 286 |
+
)
|
| 287 |
+
return bucket_id
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def _get_source_install_dependencies() -> str:
|
| 291 |
+
"""Get trackio dependencies from pyproject.toml for source installs."""
|
| 292 |
+
try:
|
| 293 |
+
import tomllib # noqa: PLC0415
|
| 294 |
+
except ModuleNotFoundError:
|
| 295 |
+
import tomli as tomllib # noqa: PLC0415
|
| 296 |
+
|
| 297 |
+
trackio_path = files("trackio")
|
| 298 |
+
pyproject_path = Path(trackio_path).parent / "pyproject.toml"
|
| 299 |
+
with open(pyproject_path, "rb") as f:
|
| 300 |
+
pyproject = tomllib.load(f)
|
| 301 |
+
deps = pyproject["project"]["dependencies"]
|
| 302 |
+
spaces_deps = (
|
| 303 |
+
pyproject["project"].get("optional-dependencies", {}).get("spaces", [])
|
| 304 |
+
)
|
| 305 |
+
mcp_deps = pyproject["project"].get("optional-dependencies", {}).get("mcp", [])
|
| 306 |
+
return "\n".join(deps + spaces_deps + mcp_deps)
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
def _get_space_install_requirement() -> str:
|
| 310 |
+
return f"trackio[spaces,mcp]=={trackio.__version__}"
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def _is_trackio_installed_from_source() -> bool:
|
| 314 |
+
"""Check if trackio is installed from source/editable install vs PyPI."""
|
| 315 |
+
try:
|
| 316 |
+
trackio_file = trackio.__file__
|
| 317 |
+
if "site-packages" not in trackio_file and "dist-packages" not in trackio_file:
|
| 318 |
+
return True
|
| 319 |
+
|
| 320 |
+
dist = importlib.metadata.distribution("trackio")
|
| 321 |
+
if dist.files:
|
| 322 |
+
files = list(dist.files)
|
| 323 |
+
has_pth = any(".pth" in str(f) for f in files)
|
| 324 |
+
if has_pth:
|
| 325 |
+
return True
|
| 326 |
+
|
| 327 |
+
return False
|
| 328 |
+
except (
|
| 329 |
+
AttributeError,
|
| 330 |
+
importlib.metadata.PackageNotFoundError,
|
| 331 |
+
importlib.metadata.MetadataError,
|
| 332 |
+
ValueError,
|
| 333 |
+
TypeError,
|
| 334 |
+
):
|
| 335 |
+
return True
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
def _upload_source_tree(
|
| 339 |
+
hf_api: huggingface_hub.HfApi, space_id: str, parent_commit: str | None = None
|
| 340 |
+
) -> huggingface_hub.CommitInfo:
|
| 341 |
+
return hf_api.upload_folder(
|
| 342 |
+
repo_id=space_id,
|
| 343 |
+
repo_type="space",
|
| 344 |
+
folder_path=files("trackio"),
|
| 345 |
+
path_in_repo="trackio",
|
| 346 |
+
parent_commit=parent_commit,
|
| 347 |
+
ignore_patterns=[
|
| 348 |
+
"README.md",
|
| 349 |
+
"frontend/node_modules/**",
|
| 350 |
+
"frontend/src/**",
|
| 351 |
+
"frontend/.gitignore",
|
| 352 |
+
"frontend/package.json",
|
| 353 |
+
"frontend/package-lock.json",
|
| 354 |
+
"frontend/vite.config.js",
|
| 355 |
+
"frontend/svelte.config.js",
|
| 356 |
+
"**/__pycache__/**",
|
| 357 |
+
"*.pyc",
|
| 358 |
+
],
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def deploy_as_space(
|
| 363 |
+
space_id: str,
|
| 364 |
+
space_storage: huggingface_hub.SpaceStorage | None = None,
|
| 365 |
+
dataset_id: str | None = None,
|
| 366 |
+
bucket_id: str | None = None,
|
| 367 |
+
private: bool | None = None,
|
| 368 |
+
frontend_dir: str | Path | None = None,
|
| 369 |
+
):
|
| 370 |
+
if on_spaces(): # in case a repo with this function is uploaded to spaces
|
| 371 |
+
return
|
| 372 |
+
|
| 373 |
+
if dataset_id is not None and bucket_id is not None:
|
| 374 |
+
raise ValueError(
|
| 375 |
+
"Cannot use bucket volume options together with dataset_id; use one persistence mode."
|
| 376 |
+
)
|
| 377 |
+
if dataset_id is not None:
|
| 378 |
+
warn_dataset_persistence_deprecated()
|
| 379 |
+
|
| 380 |
+
trackio_path = files("trackio")
|
| 381 |
+
|
| 382 |
+
hf_api = huggingface_hub.HfApi()
|
| 383 |
+
|
| 384 |
+
try:
|
| 385 |
+
huggingface_hub.create_repo(
|
| 386 |
+
space_id,
|
| 387 |
+
private=private,
|
| 388 |
+
space_sdk="gradio",
|
| 389 |
+
space_storage=space_storage,
|
| 390 |
+
repo_type="space",
|
| 391 |
+
exist_ok=True,
|
| 392 |
+
)
|
| 393 |
+
except HfHubHTTPError as e:
|
| 394 |
+
if e.response.status_code in [401, 403]: # unauthorized or forbidden
|
| 395 |
+
print("Need 'write' access token to create a Spaces repo.")
|
| 396 |
+
huggingface_hub.login(add_to_git_credential=False)
|
| 397 |
+
huggingface_hub.create_repo(
|
| 398 |
+
space_id,
|
| 399 |
+
private=private,
|
| 400 |
+
space_sdk="gradio",
|
| 401 |
+
space_storage=space_storage,
|
| 402 |
+
repo_type="space",
|
| 403 |
+
exist_ok=True,
|
| 404 |
+
)
|
| 405 |
+
else:
|
| 406 |
+
raise ValueError(f"Failed to create Space: {e}")
|
| 407 |
+
|
| 408 |
+
# We can assume huggingface-hub is available; requirements.txt pins trackio.
|
| 409 |
+
# Make sure necessary dependencies are installed by creating a requirements.txt.
|
| 410 |
+
is_source_install = _is_trackio_installed_from_source()
|
| 411 |
+
resolved_frontend = resolve_frontend_dir(frontend_dir, announce=True)
|
| 412 |
+
|
| 413 |
+
if bucket_id is not None:
|
| 414 |
+
create_bucket_if_not_exists(bucket_id, private=private)
|
| 415 |
+
|
| 416 |
+
with open(Path(trackio_path, "README.md"), "r", encoding="utf-8") as f:
|
| 417 |
+
readme_content = f.read()
|
| 418 |
+
readme_content = readme_content.replace("sdk_version: {GRADIO_VERSION}\n", "")
|
| 419 |
+
readme_content = readme_content.replace("{APP_FILE}", "app.py")
|
| 420 |
+
readme_content = readme_content.replace(
|
| 421 |
+
"{LINKED_HUB_METADATA}", _readme_linked_hub_yaml(dataset_id)
|
| 422 |
+
)
|
| 423 |
+
readme_buffer = io.BytesIO(readme_content.encode("utf-8"))
|
| 424 |
+
hf_api.upload_file(
|
| 425 |
+
path_or_fileobj=readme_buffer,
|
| 426 |
+
path_in_repo="README.md",
|
| 427 |
+
repo_id=space_id,
|
| 428 |
+
repo_type="space",
|
| 429 |
+
)
|
| 430 |
+
|
| 431 |
+
if is_source_install:
|
| 432 |
+
requirements_content = _get_source_install_dependencies()
|
| 433 |
+
else:
|
| 434 |
+
requirements_content = _get_space_install_requirement()
|
| 435 |
+
|
| 436 |
+
requirements_buffer = io.BytesIO(requirements_content.encode("utf-8"))
|
| 437 |
+
hf_api.upload_file(
|
| 438 |
+
path_or_fileobj=requirements_buffer,
|
| 439 |
+
path_in_repo="requirements.txt",
|
| 440 |
+
repo_id=space_id,
|
| 441 |
+
repo_type="space",
|
| 442 |
+
)
|
| 443 |
+
|
| 444 |
+
huggingface_hub.utils.disable_progress_bars()
|
| 445 |
+
|
| 446 |
+
if is_source_install:
|
| 447 |
+
dist_index = (
|
| 448 |
+
Path(trackio.__file__).resolve().parent / "frontend" / "dist" / "index.html"
|
| 449 |
+
)
|
| 450 |
+
if not dist_index.is_file() and not resolved_frontend.is_custom:
|
| 451 |
+
raise ValueError(
|
| 452 |
+
"The Trackio frontend build is missing. From the repository root run "
|
| 453 |
+
"`cd trackio/frontend && npm ci && npm run build`, then deploy again."
|
| 454 |
+
)
|
| 455 |
+
_upload_source_tree(hf_api, space_id)
|
| 456 |
+
|
| 457 |
+
if resolved_frontend.is_custom:
|
| 458 |
+
_upload_frontend_folder(
|
| 459 |
+
hf_api,
|
| 460 |
+
repo_id=space_id,
|
| 461 |
+
repo_type="space",
|
| 462 |
+
folder_path=resolved_frontend.path,
|
| 463 |
+
path_in_repo=_CUSTOM_SPACE_FRONTEND_DIR,
|
| 464 |
+
)
|
| 465 |
+
|
| 466 |
+
app_file_content = _space_app_py(
|
| 467 |
+
_CUSTOM_SPACE_FRONTEND_DIR if resolved_frontend.is_custom else None
|
| 468 |
+
)
|
| 469 |
+
app_file_buffer = io.BytesIO(app_file_content.encode("utf-8"))
|
| 470 |
+
hf_api.upload_file(
|
| 471 |
+
path_or_fileobj=app_file_buffer,
|
| 472 |
+
path_in_repo="app.py",
|
| 473 |
+
repo_id=space_id,
|
| 474 |
+
repo_type="space",
|
| 475 |
+
)
|
| 476 |
+
|
| 477 |
+
if hf_token := huggingface_hub.utils.get_token():
|
| 478 |
+
huggingface_hub.add_space_secret(space_id, "HF_TOKEN", hf_token)
|
| 479 |
+
if bucket_id is not None:
|
| 480 |
+
_ensure_bucket_mounted_at_data(space_id, bucket_id, hf_api)
|
| 481 |
+
elif dataset_id is not None:
|
| 482 |
+
huggingface_hub.add_space_variable(space_id, "TRACKIO_DATASET_ID", dataset_id)
|
| 483 |
+
if logo_light_url := os.environ.get("TRACKIO_LOGO_LIGHT_URL"):
|
| 484 |
+
huggingface_hub.add_space_variable(
|
| 485 |
+
space_id, "TRACKIO_LOGO_LIGHT_URL", logo_light_url
|
| 486 |
+
)
|
| 487 |
+
if logo_dark_url := os.environ.get("TRACKIO_LOGO_DARK_URL"):
|
| 488 |
+
huggingface_hub.add_space_variable(
|
| 489 |
+
space_id, "TRACKIO_LOGO_DARK_URL", logo_dark_url
|
| 490 |
+
)
|
| 491 |
+
if plot_order := os.environ.get("TRACKIO_PLOT_ORDER"):
|
| 492 |
+
huggingface_hub.add_space_variable(space_id, "TRACKIO_PLOT_ORDER", plot_order)
|
| 493 |
+
if theme := os.environ.get("TRACKIO_THEME"):
|
| 494 |
+
huggingface_hub.add_space_variable(space_id, "TRACKIO_THEME", theme)
|
| 495 |
+
huggingface_hub.add_space_variable(space_id, "GRADIO_MCP_SERVER", "True")
|
| 496 |
+
|
| 497 |
+
|
| 498 |
+
def create_space_if_not_exists(
|
| 499 |
+
space_id: str,
|
| 500 |
+
space_storage: huggingface_hub.SpaceStorage | None = None,
|
| 501 |
+
dataset_id: str | None = None,
|
| 502 |
+
bucket_id: str | None = None,
|
| 503 |
+
private: bool | None = None,
|
| 504 |
+
frontend_dir: str | Path | None = None,
|
| 505 |
+
) -> bool:
|
| 506 |
+
"""
|
| 507 |
+
Creates a new Hugging Face Space if it does not exist. Returns True if a
|
| 508 |
+
new Space was created, False if an existing Space was found.
|
| 509 |
+
|
| 510 |
+
Args:
|
| 511 |
+
space_id (`str`):
|
| 512 |
+
The ID of the Space to create.
|
| 513 |
+
space_storage ([`~huggingface_hub.SpaceStorage`], *optional*):
|
| 514 |
+
Choice of persistent storage tier for the Space.
|
| 515 |
+
dataset_id (`str`, *optional*):
|
| 516 |
+
Deprecated: persisting trackio data to a Hugging Face Dataset will be
|
| 517 |
+
removed in a future version of trackio. Use `bucket_id` (a Hugging
|
| 518 |
+
Face Bucket) instead.
|
| 519 |
+
bucket_id (`str`, *optional*):
|
| 520 |
+
Full Hub bucket id (`namespace/name`) to attach via the Hub volumes API (platform mount).
|
| 521 |
+
Sets `TRACKIO_DIR` to the mount path.
|
| 522 |
+
private (`bool`, *optional*):
|
| 523 |
+
Whether to make the Space private. If `None` (default), the repo will be
|
| 524 |
+
public unless the organization's default is private. This value is ignored
|
| 525 |
+
if the repo already exists.
|
| 526 |
+
"""
|
| 527 |
+
if "/" not in space_id:
|
| 528 |
+
raise ValueError(
|
| 529 |
+
f"Invalid space ID: {space_id}. Must be in the format: username/reponame or orgname/reponame."
|
| 530 |
+
)
|
| 531 |
+
if dataset_id is not None and "/" not in dataset_id:
|
| 532 |
+
raise ValueError(
|
| 533 |
+
f"Invalid dataset ID: {dataset_id}. Must be in the format: username/datasetname or orgname/datasetname."
|
| 534 |
+
)
|
| 535 |
+
if bucket_id is not None and "/" not in bucket_id:
|
| 536 |
+
raise ValueError(
|
| 537 |
+
f"Invalid bucket ID: {bucket_id}. Must be in the format: username/bucketname or orgname/bucketname."
|
| 538 |
+
)
|
| 539 |
+
if dataset_id is not None:
|
| 540 |
+
warn_dataset_persistence_deprecated()
|
| 541 |
+
try:
|
| 542 |
+
huggingface_hub.repo_info(space_id, repo_type="space")
|
| 543 |
+
print(
|
| 544 |
+
f"* Found existing space: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
|
| 545 |
+
)
|
| 546 |
+
if bucket_id is not None:
|
| 547 |
+
create_bucket_if_not_exists(bucket_id, private=private)
|
| 548 |
+
_ensure_bucket_mounted_at_data(space_id, bucket_id)
|
| 549 |
+
elif dataset_id is not None:
|
| 550 |
+
huggingface_hub.add_space_variable(
|
| 551 |
+
space_id, "TRACKIO_DATASET_ID", dataset_id
|
| 552 |
+
)
|
| 553 |
+
resolved_frontend = resolve_frontend_dir(frontend_dir, announce=False)
|
| 554 |
+
if resolved_frontend.is_custom:
|
| 555 |
+
deploy_as_space(
|
| 556 |
+
space_id,
|
| 557 |
+
space_storage,
|
| 558 |
+
dataset_id,
|
| 559 |
+
bucket_id,
|
| 560 |
+
private,
|
| 561 |
+
frontend_dir=frontend_dir,
|
| 562 |
+
)
|
| 563 |
+
return False
|
| 564 |
+
except RepositoryNotFoundError:
|
| 565 |
+
pass
|
| 566 |
+
except HfHubHTTPError as e:
|
| 567 |
+
if e.response.status_code in [401, 403]: # unauthorized or forbidden
|
| 568 |
+
print("Need 'write' access token to create a Spaces repo.")
|
| 569 |
+
huggingface_hub.login(add_to_git_credential=False)
|
| 570 |
+
else:
|
| 571 |
+
raise ValueError(f"Failed to create Space: {e}")
|
| 572 |
+
|
| 573 |
+
print(
|
| 574 |
+
f"* Creating new space: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
|
| 575 |
+
)
|
| 576 |
+
deploy_as_space(
|
| 577 |
+
space_id,
|
| 578 |
+
space_storage,
|
| 579 |
+
dataset_id,
|
| 580 |
+
bucket_id,
|
| 581 |
+
private,
|
| 582 |
+
frontend_dir=frontend_dir,
|
| 583 |
+
)
|
| 584 |
+
threading.Thread(
|
| 585 |
+
target=_warn_if_space_build_fails, args=(space_id,), daemon=True
|
| 586 |
+
).start()
|
| 587 |
+
return True
|
| 588 |
+
|
| 589 |
+
|
| 590 |
+
def space_is_running(space_id: str) -> bool:
|
| 591 |
+
try:
|
| 592 |
+
info = huggingface_hub.HfApi().space_info(space_id, timeout=30)
|
| 593 |
+
return bool(info.runtime) and str(info.runtime.stage) == "RUNNING"
|
| 594 |
+
except Exception:
|
| 595 |
+
return False
|
| 596 |
+
|
| 597 |
+
|
| 598 |
+
def _warn_if_space_build_fails(space_id: str, timeout: int = 600) -> None:
|
| 599 |
+
hf_api = huggingface_hub.HfApi()
|
| 600 |
+
failure_stages = frozenset(
|
| 601 |
+
("NO_APP_FILE", "CONFIG_ERROR", "BUILD_ERROR", "RUNTIME_ERROR")
|
| 602 |
+
)
|
| 603 |
+
start = time.time()
|
| 604 |
+
while time.time() - start < timeout:
|
| 605 |
+
try:
|
| 606 |
+
info = hf_api.space_info(space_id, timeout=30)
|
| 607 |
+
if info.runtime:
|
| 608 |
+
stage = str(info.runtime.stage)
|
| 609 |
+
if stage in failure_stages:
|
| 610 |
+
print(
|
| 611 |
+
f"\n* WARNING: Space {space_id} failed to build (stage: {stage}). "
|
| 612 |
+
f"Logs are being saved to the Bucket and will appear once the Space "
|
| 613 |
+
f"is fixed; see the build logs at {SPACE_URL.format(space_id=space_id)}"
|
| 614 |
+
)
|
| 615 |
+
return
|
| 616 |
+
if stage == "RUNNING":
|
| 617 |
+
return
|
| 618 |
+
except Exception:
|
| 619 |
+
pass
|
| 620 |
+
time.sleep(10)
|
| 621 |
+
|
| 622 |
+
|
| 623 |
+
def _wait_until_space_running(space_id: str, timeout: int = 300) -> None:
|
| 624 |
+
hf_api = huggingface_hub.HfApi()
|
| 625 |
+
start = time.time()
|
| 626 |
+
delay = 2
|
| 627 |
+
request_timeout = 45.0
|
| 628 |
+
failure_stages = frozenset(
|
| 629 |
+
("NO_APP_FILE", "CONFIG_ERROR", "BUILD_ERROR", "RUNTIME_ERROR")
|
| 630 |
+
)
|
| 631 |
+
while time.time() - start < timeout:
|
| 632 |
+
try:
|
| 633 |
+
info = hf_api.space_info(space_id, timeout=request_timeout)
|
| 634 |
+
if info.runtime:
|
| 635 |
+
stage = str(info.runtime.stage)
|
| 636 |
+
if stage in failure_stages:
|
| 637 |
+
raise RuntimeError(
|
| 638 |
+
f"Space {space_id} entered terminal stage {stage}. "
|
| 639 |
+
"Fix README.md or app files; see build logs on the Hub."
|
| 640 |
+
)
|
| 641 |
+
if stage == "RUNNING":
|
| 642 |
+
return
|
| 643 |
+
except RuntimeError:
|
| 644 |
+
raise
|
| 645 |
+
except (huggingface_hub.utils.HfHubHTTPError, httpx.RequestError):
|
| 646 |
+
pass
|
| 647 |
+
time.sleep(delay)
|
| 648 |
+
delay = min(delay * 1.5, 15)
|
| 649 |
+
raise TimeoutError(
|
| 650 |
+
f"Space {space_id} did not reach RUNNING within {timeout}s. "
|
| 651 |
+
"Check status and build logs on the Hub."
|
| 652 |
+
)
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
def wait_until_space_exists(
|
| 656 |
+
space_id: str,
|
| 657 |
+
) -> None:
|
| 658 |
+
"""
|
| 659 |
+
Blocks the current thread until the Space exists.
|
| 660 |
+
|
| 661 |
+
Args:
|
| 662 |
+
space_id (`str`):
|
| 663 |
+
The ID of the Space to wait for.
|
| 664 |
+
|
| 665 |
+
Raises:
|
| 666 |
+
`TimeoutError`: If waiting for the Space takes longer than expected.
|
| 667 |
+
"""
|
| 668 |
+
hf_api = huggingface_hub.HfApi()
|
| 669 |
+
delay = 1
|
| 670 |
+
for _ in range(30):
|
| 671 |
+
try:
|
| 672 |
+
hf_api.space_info(space_id)
|
| 673 |
+
return
|
| 674 |
+
except (huggingface_hub.utils.HfHubHTTPError, httpx.RequestError):
|
| 675 |
+
time.sleep(delay)
|
| 676 |
+
delay = min(delay * 2, 60)
|
| 677 |
+
raise TimeoutError("Waiting for space to exist took longer than expected")
|
| 678 |
+
|
| 679 |
+
|
| 680 |
+
def upload_db_to_space(project: str, space_id: str, force: bool = False) -> None:
|
| 681 |
+
"""
|
| 682 |
+
Uploads the database of a local Trackio project to a Hugging Face Space.
|
| 683 |
+
|
| 684 |
+
This uses the Trackio remote client so newer Trackio Spaces can speak the direct
|
| 685 |
+
HTTP API while older Gradio-based Spaces still work through `gradio_client`.
|
| 686 |
+
|
| 687 |
+
Args:
|
| 688 |
+
project (`str`):
|
| 689 |
+
The name of the project to upload.
|
| 690 |
+
space_id (`str`):
|
| 691 |
+
The ID of the Space to upload to.
|
| 692 |
+
force (`bool`, *optional*, defaults to `False`):
|
| 693 |
+
If `True`, overwrites the existing database without prompting. If `False`,
|
| 694 |
+
prompts for confirmation.
|
| 695 |
+
"""
|
| 696 |
+
db_path = SQLiteStorage.get_project_db_path(project)
|
| 697 |
+
client = RemoteClient(
|
| 698 |
+
space_id,
|
| 699 |
+
hf_token=huggingface_hub.utils.get_token(),
|
| 700 |
+
httpx_kwargs={"timeout": 90},
|
| 701 |
+
)
|
| 702 |
+
|
| 703 |
+
if not force:
|
| 704 |
+
try:
|
| 705 |
+
existing_projects = client.predict(api_name="/get_all_projects")
|
| 706 |
+
if project in existing_projects:
|
| 707 |
+
response = input(
|
| 708 |
+
f"Database for project '{project}' already exists on Space '{space_id}'. "
|
| 709 |
+
f"Overwrite it? (y/N): "
|
| 710 |
+
)
|
| 711 |
+
if response.lower() not in ["y", "yes"]:
|
| 712 |
+
print("* Upload cancelled.")
|
| 713 |
+
return
|
| 714 |
+
except Exception as e:
|
| 715 |
+
print(f"* Warning: Could not check if project exists on Space: {e}")
|
| 716 |
+
print("* Proceeding with upload...")
|
| 717 |
+
|
| 718 |
+
client.predict(
|
| 719 |
+
api_name="/upload_db_to_space",
|
| 720 |
+
project=project,
|
| 721 |
+
uploaded_db=handle_file(db_path),
|
| 722 |
+
hf_token=huggingface_hub.utils.get_token(),
|
| 723 |
+
)
|
| 724 |
+
|
| 725 |
+
|
| 726 |
+
SYNC_BATCH_SIZE = 500
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
def _replay_pending_uploads(
|
| 730 |
+
project: str,
|
| 731 |
+
client: "RemoteClient",
|
| 732 |
+
hf_token: str | None,
|
| 733 |
+
) -> None:
|
| 734 |
+
"""Replay queued `pending_uploads` rows for `project` via `client` — the
|
| 735 |
+
out-of-Run sync path (`trackio sync`); see `replay_pending_uploads` for the
|
| 736 |
+
per-kind routing and clearing semantics.
|
| 737 |
+
"""
|
| 738 |
+
pending_uploads = SQLiteStorage.get_pending_uploads(project)
|
| 739 |
+
if not pending_uploads:
|
| 740 |
+
return
|
| 741 |
+
|
| 742 |
+
def _warn_missing(count: int, sample: str) -> None:
|
| 743 |
+
print(
|
| 744 |
+
f" Warning: dropping {count} pending upload(s) whose local files "
|
| 745 |
+
f"no longer exist (e.g. {sample!r})."
|
| 746 |
+
)
|
| 747 |
+
|
| 748 |
+
replay_pending_uploads(
|
| 749 |
+
pending_uploads,
|
| 750 |
+
project,
|
| 751 |
+
predict=client.predict,
|
| 752 |
+
hf_token=hf_token,
|
| 753 |
+
warn_missing=_warn_missing,
|
| 754 |
+
verbose=True,
|
| 755 |
+
)
|
| 756 |
+
|
| 757 |
+
|
| 758 |
+
def sync_incremental(
|
| 759 |
+
project: str,
|
| 760 |
+
space_id: str,
|
| 761 |
+
private: bool | None = None,
|
| 762 |
+
pending_only: bool = False,
|
| 763 |
+
frontend_dir: str | Path | None = None,
|
| 764 |
+
) -> None:
|
| 765 |
+
"""
|
| 766 |
+
Syncs a local Trackio project to a Space via the bulk_log API endpoints
|
| 767 |
+
instead of uploading the entire DB file. Supports incremental sync.
|
| 768 |
+
|
| 769 |
+
Args:
|
| 770 |
+
project: The name of the project to sync.
|
| 771 |
+
space_id: The HF Space ID to sync to.
|
| 772 |
+
private: Whether to make the Space private if creating.
|
| 773 |
+
pending_only: If True, only sync rows tagged with space_id (pending data).
|
| 774 |
+
"""
|
| 775 |
+
print(
|
| 776 |
+
f"* Syncing project '{project}' to: {SPACE_URL.format(space_id=space_id)} (please wait...)"
|
| 777 |
+
)
|
| 778 |
+
create_space_if_not_exists(space_id, private=private, frontend_dir=frontend_dir)
|
| 779 |
+
wait_until_space_exists(space_id)
|
| 780 |
+
hf_token = huggingface_hub.utils.get_token()
|
| 781 |
+
expected_run_counts: Counter[str] = Counter()
|
| 782 |
+
|
| 783 |
+
client = RemoteClient(
|
| 784 |
+
space_id,
|
| 785 |
+
hf_token=hf_token,
|
| 786 |
+
httpx_kwargs={"timeout": 90},
|
| 787 |
+
)
|
| 788 |
+
|
| 789 |
+
if pending_only:
|
| 790 |
+
pending_logs = SQLiteStorage.get_pending_logs(project)
|
| 791 |
+
if pending_logs:
|
| 792 |
+
logs = pending_logs["logs"]
|
| 793 |
+
expected_run_counts.update(log["run"] for log in logs)
|
| 794 |
+
for i in range(0, len(logs), SYNC_BATCH_SIZE):
|
| 795 |
+
batch = logs[i : i + SYNC_BATCH_SIZE]
|
| 796 |
+
print(
|
| 797 |
+
f" Syncing metrics: {min(i + SYNC_BATCH_SIZE, len(logs))}/{len(logs)}..."
|
| 798 |
+
)
|
| 799 |
+
client.predict(api_name="/bulk_log", logs=batch, hf_token=hf_token)
|
| 800 |
+
SQLiteStorage.clear_pending_logs(project, pending_logs["ids"])
|
| 801 |
+
|
| 802 |
+
pending_sys = SQLiteStorage.get_pending_system_logs(project)
|
| 803 |
+
if pending_sys:
|
| 804 |
+
logs = pending_sys["logs"]
|
| 805 |
+
for i in range(0, len(logs), SYNC_BATCH_SIZE):
|
| 806 |
+
batch = logs[i : i + SYNC_BATCH_SIZE]
|
| 807 |
+
print(
|
| 808 |
+
f" Syncing system metrics: {min(i + SYNC_BATCH_SIZE, len(logs))}/{len(logs)}..."
|
| 809 |
+
)
|
| 810 |
+
client.predict(
|
| 811 |
+
api_name="/bulk_log_system", logs=batch, hf_token=hf_token
|
| 812 |
+
)
|
| 813 |
+
SQLiteStorage.clear_pending_system_logs(project, pending_sys["ids"])
|
| 814 |
+
|
| 815 |
+
_replay_pending_uploads(project, client, hf_token)
|
| 816 |
+
else:
|
| 817 |
+
all_logs = SQLiteStorage.get_all_logs_for_sync(project)
|
| 818 |
+
if all_logs:
|
| 819 |
+
expected_run_counts.update(log["run"] for log in all_logs)
|
| 820 |
+
for i in range(0, len(all_logs), SYNC_BATCH_SIZE):
|
| 821 |
+
batch = all_logs[i : i + SYNC_BATCH_SIZE]
|
| 822 |
+
print(
|
| 823 |
+
f" Syncing metrics: {min(i + SYNC_BATCH_SIZE, len(all_logs))}/{len(all_logs)}..."
|
| 824 |
+
)
|
| 825 |
+
client.predict(api_name="/bulk_log", logs=batch, hf_token=hf_token)
|
| 826 |
+
|
| 827 |
+
all_sys_logs = SQLiteStorage.get_all_system_logs_for_sync(project)
|
| 828 |
+
if all_sys_logs:
|
| 829 |
+
for i in range(0, len(all_sys_logs), SYNC_BATCH_SIZE):
|
| 830 |
+
batch = all_sys_logs[i : i + SYNC_BATCH_SIZE]
|
| 831 |
+
print(
|
| 832 |
+
f" Syncing system metrics: {min(i + SYNC_BATCH_SIZE, len(all_sys_logs))}/{len(all_sys_logs)}..."
|
| 833 |
+
)
|
| 834 |
+
client.predict(
|
| 835 |
+
api_name="/bulk_log_system", logs=batch, hf_token=hf_token
|
| 836 |
+
)
|
| 837 |
+
|
| 838 |
+
_wait_for_remote_sync(client, project, expected_run_counts)
|
| 839 |
+
SQLiteStorage.set_project_metadata(project, "space_id", space_id)
|
| 840 |
+
print(
|
| 841 |
+
f"* Synced successfully to space: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
|
| 842 |
+
)
|
| 843 |
+
|
| 844 |
+
|
| 845 |
+
def _build_remote_client_with_retry(
|
| 846 |
+
space_id: str,
|
| 847 |
+
timeout: int = 360,
|
| 848 |
+
verbose: bool = False,
|
| 849 |
+
) -> RemoteClient:
|
| 850 |
+
deadline = time.time() + timeout
|
| 851 |
+
delay = 2
|
| 852 |
+
last_error: Exception | None = None
|
| 853 |
+
while time.time() < deadline:
|
| 854 |
+
try:
|
| 855 |
+
return RemoteClient(
|
| 856 |
+
space_id,
|
| 857 |
+
hf_token=huggingface_hub.utils.get_token(),
|
| 858 |
+
verbose=verbose,
|
| 859 |
+
httpx_kwargs={"timeout": 90},
|
| 860 |
+
)
|
| 861 |
+
except (ValueError, ConnectionError) as e:
|
| 862 |
+
last_error = e
|
| 863 |
+
time.sleep(delay)
|
| 864 |
+
delay = min(delay * 1.5, 15)
|
| 865 |
+
raise ConnectionError(
|
| 866 |
+
f"Could not connect to Space '{space_id}' within {timeout}s: {last_error}"
|
| 867 |
+
)
|
| 868 |
+
|
| 869 |
+
|
| 870 |
+
def _wait_for_remote_sync(
|
| 871 |
+
client: RemoteClient,
|
| 872 |
+
project: str,
|
| 873 |
+
expected_run_counts: Counter[str],
|
| 874 |
+
timeout: int = 180,
|
| 875 |
+
) -> None:
|
| 876 |
+
if not expected_run_counts:
|
| 877 |
+
return
|
| 878 |
+
|
| 879 |
+
deadline = time.time() + timeout
|
| 880 |
+
delay = 2
|
| 881 |
+
last_error: Exception | None = None
|
| 882 |
+
pending = dict(expected_run_counts)
|
| 883 |
+
|
| 884 |
+
while time.time() < deadline and pending:
|
| 885 |
+
completed = []
|
| 886 |
+
for run_name, expected_num_logs in pending.items():
|
| 887 |
+
try:
|
| 888 |
+
summary = client.predict(
|
| 889 |
+
project=project, run=run_name, api_name="/get_run_summary"
|
| 890 |
+
)
|
| 891 |
+
if summary.get("num_logs") == expected_num_logs:
|
| 892 |
+
completed.append(run_name)
|
| 893 |
+
except Exception as e:
|
| 894 |
+
last_error = e
|
| 895 |
+
for run_name in completed:
|
| 896 |
+
pending.pop(run_name, None)
|
| 897 |
+
if pending:
|
| 898 |
+
time.sleep(delay)
|
| 899 |
+
delay = min(delay * 1.5, 15)
|
| 900 |
+
|
| 901 |
+
if pending:
|
| 902 |
+
raise TimeoutError(
|
| 903 |
+
f"Remote sync for project '{project}' did not become visible for runs "
|
| 904 |
+
f"{sorted(pending.items())} within {timeout}s. "
|
| 905 |
+
f"Last error: {last_error!r}"
|
| 906 |
+
)
|
| 907 |
+
|
| 908 |
+
|
| 909 |
+
def upload_dataset_for_static(
|
| 910 |
+
project: str,
|
| 911 |
+
dataset_id: str,
|
| 912 |
+
private: bool | None = None,
|
| 913 |
+
) -> None:
|
| 914 |
+
hf_api = huggingface_hub.HfApi()
|
| 915 |
+
|
| 916 |
+
try:
|
| 917 |
+
huggingface_hub.create_repo(
|
| 918 |
+
dataset_id,
|
| 919 |
+
private=private,
|
| 920 |
+
repo_type="dataset",
|
| 921 |
+
exist_ok=True,
|
| 922 |
+
)
|
| 923 |
+
except HfHubHTTPError as e:
|
| 924 |
+
if e.response.status_code in [401, 403]:
|
| 925 |
+
print("Need 'write' access token to create a Dataset repo.")
|
| 926 |
+
huggingface_hub.login(add_to_git_credential=False)
|
| 927 |
+
huggingface_hub.create_repo(
|
| 928 |
+
dataset_id,
|
| 929 |
+
private=private,
|
| 930 |
+
repo_type="dataset",
|
| 931 |
+
exist_ok=True,
|
| 932 |
+
)
|
| 933 |
+
else:
|
| 934 |
+
raise ValueError(f"Failed to create Dataset: {e}")
|
| 935 |
+
|
| 936 |
+
def _upload_folder(folder: Path, path_in_repo: str, ignore_patterns=None) -> None:
|
| 937 |
+
if not folder.exists():
|
| 938 |
+
return
|
| 939 |
+
_retry_hf_write(
|
| 940 |
+
f"Dataset {path_in_repo} upload",
|
| 941 |
+
lambda: hf_api.upload_folder(
|
| 942 |
+
repo_id=dataset_id,
|
| 943 |
+
repo_type="dataset",
|
| 944 |
+
folder_path=str(folder),
|
| 945 |
+
path_in_repo=path_in_repo,
|
| 946 |
+
ignore_patterns=ignore_patterns,
|
| 947 |
+
),
|
| 948 |
+
)
|
| 949 |
+
|
| 950 |
+
with tempfile.TemporaryDirectory() as tmp_dir:
|
| 951 |
+
output_dir = Path(tmp_dir)
|
| 952 |
+
SQLiteStorage.export_for_static_space(project, output_dir)
|
| 953 |
+
|
| 954 |
+
_upload_folder(project_media_dir(project), "media")
|
| 955 |
+
_upload_folder(project_artifacts_dir(project), "artifacts", [PARTIAL_BLOB_GLOB])
|
| 956 |
+
|
| 957 |
+
_retry_hf_write(
|
| 958 |
+
"Dataset upload",
|
| 959 |
+
lambda: hf_api.upload_folder(
|
| 960 |
+
repo_id=dataset_id,
|
| 961 |
+
repo_type="dataset",
|
| 962 |
+
folder_path=str(output_dir),
|
| 963 |
+
),
|
| 964 |
+
)
|
| 965 |
+
|
| 966 |
+
print(f"* Dataset uploaded: https://huggingface.co/datasets/{dataset_id}")
|
| 967 |
+
|
| 968 |
+
|
| 969 |
+
def deploy_as_static_space(
|
| 970 |
+
space_id: str,
|
| 971 |
+
dataset_id: str | None,
|
| 972 |
+
project: str,
|
| 973 |
+
bucket_id: str | None = None,
|
| 974 |
+
private: bool | None = None,
|
| 975 |
+
hf_token: str | None = None,
|
| 976 |
+
frontend_dir: str | Path | None = None,
|
| 977 |
+
) -> None:
|
| 978 |
+
if on_spaces():
|
| 979 |
+
return
|
| 980 |
+
|
| 981 |
+
if private is True:
|
| 982 |
+
raise ValueError(
|
| 983 |
+
"private=True is not supported for static Trackio Spaces. Static Spaces "
|
| 984 |
+
"run entirely in the browser, so their snapshot data must be public. "
|
| 985 |
+
"Use sdk='gradio' for a private dashboard."
|
| 986 |
+
)
|
| 987 |
+
if dataset_id is not None:
|
| 988 |
+
warn_dataset_persistence_deprecated()
|
| 989 |
+
hf_api = huggingface_hub.HfApi()
|
| 990 |
+
|
| 991 |
+
try:
|
| 992 |
+
huggingface_hub.create_repo(
|
| 993 |
+
space_id,
|
| 994 |
+
private=False,
|
| 995 |
+
space_sdk="static",
|
| 996 |
+
repo_type="space",
|
| 997 |
+
exist_ok=True,
|
| 998 |
+
)
|
| 999 |
+
except HfHubHTTPError as e:
|
| 1000 |
+
if e.response.status_code in [401, 403]:
|
| 1001 |
+
print("Need 'write' access token to create a Spaces repo.")
|
| 1002 |
+
huggingface_hub.login(add_to_git_credential=False)
|
| 1003 |
+
huggingface_hub.create_repo(
|
| 1004 |
+
space_id,
|
| 1005 |
+
private=False,
|
| 1006 |
+
space_sdk="static",
|
| 1007 |
+
repo_type="space",
|
| 1008 |
+
exist_ok=True,
|
| 1009 |
+
)
|
| 1010 |
+
else:
|
| 1011 |
+
raise ValueError(f"Failed to create Space: {e}")
|
| 1012 |
+
|
| 1013 |
+
linked = _readme_linked_hub_yaml(dataset_id)
|
| 1014 |
+
readme_content = (
|
| 1015 |
+
f"---\nemoji: 🎯\nsdk: static\npinned: false\ntags:\n - trackio\n{linked}---\n"
|
| 1016 |
+
)
|
| 1017 |
+
_retry_hf_write(
|
| 1018 |
+
"Static Space README upload",
|
| 1019 |
+
lambda: hf_api.upload_file(
|
| 1020 |
+
path_or_fileobj=io.BytesIO(readme_content.encode("utf-8")),
|
| 1021 |
+
path_in_repo="README.md",
|
| 1022 |
+
repo_id=space_id,
|
| 1023 |
+
repo_type="space",
|
| 1024 |
+
),
|
| 1025 |
+
)
|
| 1026 |
+
|
| 1027 |
+
resolved_frontend = resolve_frontend_dir(frontend_dir, announce=True)
|
| 1028 |
+
|
| 1029 |
+
_retry_hf_write(
|
| 1030 |
+
"Static Space frontend upload",
|
| 1031 |
+
lambda: hf_api.upload_folder(
|
| 1032 |
+
repo_id=space_id,
|
| 1033 |
+
repo_type="space",
|
| 1034 |
+
folder_path=str(resolved_frontend.path),
|
| 1035 |
+
),
|
| 1036 |
+
)
|
| 1037 |
+
|
| 1038 |
+
config = {
|
| 1039 |
+
"mode": "static",
|
| 1040 |
+
"project": project,
|
| 1041 |
+
"private": bool(private),
|
| 1042 |
+
"version": trackio.__version__,
|
| 1043 |
+
}
|
| 1044 |
+
if bucket_id is not None:
|
| 1045 |
+
config["bucket_id"] = bucket_id
|
| 1046 |
+
if dataset_id is not None:
|
| 1047 |
+
config["dataset_id"] = dataset_id
|
| 1048 |
+
if hf_token is not None:
|
| 1049 |
+
warnings.warn(
|
| 1050 |
+
"`hf_token` is ignored by deploy_as_static_space() for static Space "
|
| 1051 |
+
"deployment and will be removed in a future release.",
|
| 1052 |
+
DeprecationWarning,
|
| 1053 |
+
stacklevel=2,
|
| 1054 |
+
)
|
| 1055 |
+
|
| 1056 |
+
_retry_hf_write(
|
| 1057 |
+
"Static Space config upload",
|
| 1058 |
+
lambda: hf_api.upload_file(
|
| 1059 |
+
path_or_fileobj=io.BytesIO(json_mod.dumps(config).encode("utf-8")),
|
| 1060 |
+
path_in_repo="config.json",
|
| 1061 |
+
repo_id=space_id,
|
| 1062 |
+
repo_type="space",
|
| 1063 |
+
),
|
| 1064 |
+
)
|
| 1065 |
+
|
| 1066 |
+
assets_dir = Path(trackio.__file__).resolve().parent / "assets"
|
| 1067 |
+
if assets_dir.is_dir():
|
| 1068 |
+
_retry_hf_write(
|
| 1069 |
+
"Static Space assets upload",
|
| 1070 |
+
lambda: hf_api.upload_folder(
|
| 1071 |
+
repo_id=space_id,
|
| 1072 |
+
repo_type="space",
|
| 1073 |
+
folder_path=str(assets_dir),
|
| 1074 |
+
path_in_repo="static/trackio",
|
| 1075 |
+
),
|
| 1076 |
+
)
|
| 1077 |
+
|
| 1078 |
+
print(
|
| 1079 |
+
f"* Static Space deployed: {_BOLD_ORANGE}{SPACE_URL.format(space_id=space_id)}{_RESET}"
|
| 1080 |
+
)
|
| 1081 |
+
|
| 1082 |
+
|
| 1083 |
+
def sync(
|
| 1084 |
+
project: str,
|
| 1085 |
+
space_id: str | None = None,
|
| 1086 |
+
private: bool | None = None,
|
| 1087 |
+
force: bool = False,
|
| 1088 |
+
run_in_background: bool = False,
|
| 1089 |
+
sdk: str = "gradio",
|
| 1090 |
+
dataset_id: str | None = None,
|
| 1091 |
+
bucket_id: str | None = None,
|
| 1092 |
+
frontend_dir: str | Path | None = None,
|
| 1093 |
+
) -> str:
|
| 1094 |
+
"""
|
| 1095 |
+
Syncs a local Trackio project's database to a Hugging Face Space.
|
| 1096 |
+
If the Space does not exist, it will be created. Local data is never deleted.
|
| 1097 |
+
|
| 1098 |
+
**Freezing:** Passing ``sdk="static"`` deploys a static Space backed by an HF Bucket
|
| 1099 |
+
(read-only dashboard, no Gradio server). You can sync the same project again later to
|
| 1100 |
+
refresh that static Space. If you want a one-time snapshot of an existing Gradio Space,
|
| 1101 |
+
use ``freeze()`` instead.
|
| 1102 |
+
|
| 1103 |
+
Args:
|
| 1104 |
+
project (`str`): The name of the project to upload.
|
| 1105 |
+
space_id (`str`, *optional*): The ID of the Space to upload to (e.g., `"username/space_id"`).
|
| 1106 |
+
If not provided, checks project metadata first, then generates a random space_id.
|
| 1107 |
+
private (`bool`, *optional*):
|
| 1108 |
+
Whether to make the Space private. If None (default), the repo will be
|
| 1109 |
+
public unless the organization's default is private. This value is ignored
|
| 1110 |
+
if the repo already exists. Not supported with ``sdk="static"`` because
|
| 1111 |
+
static Trackio dashboards read snapshot data directly from the browser.
|
| 1112 |
+
force (`bool`, *optional*, defaults to `False`):
|
| 1113 |
+
If `True`, overwrite the existing database without prompting for confirmation.
|
| 1114 |
+
If `False`, prompt the user before overwriting an existing database.
|
| 1115 |
+
run_in_background (`bool`, *optional*, defaults to `False`):
|
| 1116 |
+
If `True`, the Space creation and database upload will be run in a background thread.
|
| 1117 |
+
If `False`, all the steps will be run synchronously.
|
| 1118 |
+
sdk (`str`, *optional*, defaults to `"gradio"`):
|
| 1119 |
+
The type of Space to deploy. `"gradio"` deploys a Gradio Space with a live
|
| 1120 |
+
server. `"static"` freezes the Space: deploys a static Space that reads from an HF Bucket
|
| 1121 |
+
(no server needed).
|
| 1122 |
+
dataset_id (`str`, *optional*):
|
| 1123 |
+
Deprecated: persisting trackio data to a Hugging Face Dataset will be
|
| 1124 |
+
removed in a future version of trackio. Use `bucket_id` (a Hugging
|
| 1125 |
+
Face Bucket) instead.
|
| 1126 |
+
bucket_id (`str`, *optional*):
|
| 1127 |
+
The ID of the HF Bucket to sync to. By default, a bucket is auto-generated
|
| 1128 |
+
from the space_id.
|
| 1129 |
+
Returns:
|
| 1130 |
+
`str`: The Space ID of the synced project.
|
| 1131 |
+
"""
|
| 1132 |
+
if sdk not in ("gradio", "static"):
|
| 1133 |
+
raise ValueError(f"sdk must be 'gradio' or 'static', got '{sdk}'")
|
| 1134 |
+
if sdk == "static" and private is True:
|
| 1135 |
+
raise ValueError(
|
| 1136 |
+
"private=True is not supported for static Trackio Spaces. Static Spaces "
|
| 1137 |
+
"run entirely in the browser, so their snapshot data must be public. "
|
| 1138 |
+
"Use sdk='gradio' for a private dashboard."
|
| 1139 |
+
)
|
| 1140 |
+
bucket_id_was_explicit = bucket_id is not None
|
| 1141 |
+
|
| 1142 |
+
if space_id is None:
|
| 1143 |
+
space_id = SQLiteStorage.get_space_id(project)
|
| 1144 |
+
if space_id is None:
|
| 1145 |
+
space_id = f"{project}-{get_or_create_project_hash(project)}"
|
| 1146 |
+
space_id, dataset_id, bucket_id = preprocess_space_and_dataset_ids(
|
| 1147 |
+
space_id, dataset_id, bucket_id
|
| 1148 |
+
)
|
| 1149 |
+
if dataset_id is None and bucket_id is not None and not bucket_id_was_explicit:
|
| 1150 |
+
bucket_id = resolve_auto_bucket_id(space_id, bucket_id)
|
| 1151 |
+
|
| 1152 |
+
def _do_sync():
|
| 1153 |
+
try:
|
| 1154 |
+
info = huggingface_hub.HfApi().space_info(space_id)
|
| 1155 |
+
existing_sdk = info.sdk
|
| 1156 |
+
if existing_sdk and existing_sdk != sdk:
|
| 1157 |
+
raise ValueError(
|
| 1158 |
+
f"Space '{space_id}' is a '{existing_sdk}' Space but sdk='{sdk}' was requested. "
|
| 1159 |
+
f"The sdk must match the existing Space type."
|
| 1160 |
+
)
|
| 1161 |
+
except RepositoryNotFoundError:
|
| 1162 |
+
pass
|
| 1163 |
+
|
| 1164 |
+
if sdk == "static":
|
| 1165 |
+
if dataset_id is not None:
|
| 1166 |
+
upload_dataset_for_static(project, dataset_id, private=False)
|
| 1167 |
+
deploy_as_static_space(
|
| 1168 |
+
space_id,
|
| 1169 |
+
dataset_id,
|
| 1170 |
+
project,
|
| 1171 |
+
private=False,
|
| 1172 |
+
frontend_dir=frontend_dir,
|
| 1173 |
+
)
|
| 1174 |
+
elif bucket_id is not None:
|
| 1175 |
+
create_bucket_if_not_exists(bucket_id, private=False)
|
| 1176 |
+
upload_project_to_bucket_for_static(project, bucket_id)
|
| 1177 |
+
print(
|
| 1178 |
+
f"* Project data uploaded to bucket: https://huggingface.co/buckets/{bucket_id}"
|
| 1179 |
+
)
|
| 1180 |
+
deploy_as_static_space(
|
| 1181 |
+
space_id,
|
| 1182 |
+
None,
|
| 1183 |
+
project,
|
| 1184 |
+
bucket_id=bucket_id,
|
| 1185 |
+
private=False,
|
| 1186 |
+
frontend_dir=frontend_dir,
|
| 1187 |
+
)
|
| 1188 |
+
else:
|
| 1189 |
+
if bucket_id is not None:
|
| 1190 |
+
create_bucket_if_not_exists(bucket_id, private=private)
|
| 1191 |
+
upload_project_to_bucket(project, bucket_id)
|
| 1192 |
+
print(
|
| 1193 |
+
f"* Project data uploaded to bucket: https://huggingface.co/buckets/{bucket_id}"
|
| 1194 |
+
)
|
| 1195 |
+
create_space_if_not_exists(
|
| 1196 |
+
space_id,
|
| 1197 |
+
bucket_id=bucket_id,
|
| 1198 |
+
private=private,
|
| 1199 |
+
frontend_dir=frontend_dir,
|
| 1200 |
+
)
|
| 1201 |
+
_wait_until_space_running(space_id)
|
| 1202 |
+
_wait_for_remote_sync(
|
| 1203 |
+
_build_remote_client_with_retry(space_id),
|
| 1204 |
+
project,
|
| 1205 |
+
Counter(
|
| 1206 |
+
log["run"]
|
| 1207 |
+
for log in SQLiteStorage.get_all_logs_for_sync(project)
|
| 1208 |
+
),
|
| 1209 |
+
)
|
| 1210 |
+
else:
|
| 1211 |
+
sync_incremental(
|
| 1212 |
+
project,
|
| 1213 |
+
space_id,
|
| 1214 |
+
private=private,
|
| 1215 |
+
pending_only=False,
|
| 1216 |
+
frontend_dir=frontend_dir,
|
| 1217 |
+
)
|
| 1218 |
+
SQLiteStorage.set_project_metadata(project, "space_id", space_id)
|
| 1219 |
+
|
| 1220 |
+
if run_in_background:
|
| 1221 |
+
threading.Thread(target=_do_sync).start()
|
| 1222 |
+
else:
|
| 1223 |
+
_do_sync()
|
| 1224 |
+
return space_id
|
| 1225 |
+
|
| 1226 |
+
|
| 1227 |
+
def _get_source_bucket(space_id: str) -> str:
|
| 1228 |
+
bucket_id = _get_existing_space_bucket(space_id)
|
| 1229 |
+
if bucket_id is not None:
|
| 1230 |
+
_ensure_bucket_mounted_at_data(space_id, bucket_id)
|
| 1231 |
+
return bucket_id
|
| 1232 |
+
raise ValueError(
|
| 1233 |
+
f"Space '{space_id}' has no bucket mounted at '/data'. "
|
| 1234 |
+
f"freeze() requires the source Space to use bucket storage."
|
| 1235 |
+
)
|
| 1236 |
+
|
| 1237 |
+
|
| 1238 |
+
def freeze(
|
| 1239 |
+
space_id: str,
|
| 1240 |
+
project: str,
|
| 1241 |
+
new_space_id: str | None = None,
|
| 1242 |
+
private: bool | None = None,
|
| 1243 |
+
bucket_id: str | None = None,
|
| 1244 |
+
frontend_dir: str | Path | None = None,
|
| 1245 |
+
) -> str:
|
| 1246 |
+
"""
|
| 1247 |
+
Creates a new static Hugging Face Space containing a read-only snapshot of
|
| 1248 |
+
the data for the specified project from the source Gradio Space. The data is
|
| 1249 |
+
read from the bucket attached to the source Space at freeze time. The original
|
| 1250 |
+
Space is not modified, and the new static Space does not automatically reflect
|
| 1251 |
+
metrics uploaded to the original Gradio Space after the freeze completes.
|
| 1252 |
+
|
| 1253 |
+
Args:
|
| 1254 |
+
space_id (`str`):
|
| 1255 |
+
The ID of the source Gradio Space (e.g., `"username/my-space"` or a
|
| 1256 |
+
short repo name with the logged-in namespace inferred, like `init()`).
|
| 1257 |
+
Must be a Gradio Space with a bucket mounted at `/data`.
|
| 1258 |
+
project (`str`):
|
| 1259 |
+
The name of the project whose data to include in the frozen Space.
|
| 1260 |
+
new_space_id (`str`, *optional*):
|
| 1261 |
+
The ID for the new static Space. If not provided, defaults to
|
| 1262 |
+
`"{space_id}_static"`.
|
| 1263 |
+
private (`bool`, *optional*):
|
| 1264 |
+
Not supported. Frozen static dashboards read snapshot data directly
|
| 1265 |
+
from the browser, so the destination snapshot must be public.
|
| 1266 |
+
bucket_id (`str`, *optional*):
|
| 1267 |
+
The ID of the HF Bucket for the new static Space's data storage.
|
| 1268 |
+
If not provided, one is auto-generated from the new Space ID.
|
| 1269 |
+
|
| 1270 |
+
Returns:
|
| 1271 |
+
`str`: The Space ID of the newly created static Space.
|
| 1272 |
+
"""
|
| 1273 |
+
if private is True:
|
| 1274 |
+
raise ValueError(
|
| 1275 |
+
"private=True is not supported for frozen static Trackio Spaces. Static "
|
| 1276 |
+
"Spaces run entirely in the browser, so their snapshot data must be "
|
| 1277 |
+
"public. Use a Gradio Space if the frozen dashboard must stay private."
|
| 1278 |
+
)
|
| 1279 |
+
space_id, _, _ = preprocess_space_and_dataset_ids(space_id, None, None)
|
| 1280 |
+
|
| 1281 |
+
try:
|
| 1282 |
+
info = huggingface_hub.HfApi().space_info(space_id)
|
| 1283 |
+
if info.sdk != "gradio":
|
| 1284 |
+
raise ValueError(
|
| 1285 |
+
f"Space '{space_id}' is not a Gradio Space (sdk='{info.sdk}'). "
|
| 1286 |
+
f"freeze() requires a Gradio Space as the source."
|
| 1287 |
+
)
|
| 1288 |
+
except RepositoryNotFoundError:
|
| 1289 |
+
raise ValueError(
|
| 1290 |
+
f"Space '{space_id}' not found. Provide an existing Gradio Space ID."
|
| 1291 |
+
)
|
| 1292 |
+
|
| 1293 |
+
source_bucket_id = _get_source_bucket(space_id)
|
| 1294 |
+
print(f"* Reading project data from bucket: {source_bucket_id}")
|
| 1295 |
+
|
| 1296 |
+
bucket_id_was_explicit = bucket_id is not None
|
| 1297 |
+
|
| 1298 |
+
if new_space_id is None:
|
| 1299 |
+
new_space_id = f"{space_id}_static"
|
| 1300 |
+
new_space_id, _dataset_id, bucket_id = preprocess_space_and_dataset_ids(
|
| 1301 |
+
new_space_id, None, bucket_id
|
| 1302 |
+
)
|
| 1303 |
+
if bucket_id is not None and not bucket_id_was_explicit:
|
| 1304 |
+
bucket_id = resolve_auto_bucket_id(new_space_id, bucket_id)
|
| 1305 |
+
|
| 1306 |
+
hf_api = huggingface_hub.HfApi()
|
| 1307 |
+
try:
|
| 1308 |
+
dest_info = hf_api.space_info(new_space_id)
|
| 1309 |
+
tags = dest_info.tags or []
|
| 1310 |
+
if dest_info.sdk != "static" or "trackio" not in tags:
|
| 1311 |
+
raise ValueError(
|
| 1312 |
+
f"Space '{new_space_id}' already exists and is not a Trackio static Space "
|
| 1313 |
+
f"(sdk='{dest_info.sdk}', tags={tags}). Choose a different new_space_id "
|
| 1314 |
+
f"or delete the existing Space first."
|
| 1315 |
+
)
|
| 1316 |
+
except RepositoryNotFoundError:
|
| 1317 |
+
pass
|
| 1318 |
+
|
| 1319 |
+
create_bucket_if_not_exists(bucket_id, private=False)
|
| 1320 |
+
export_from_bucket_for_static(source_bucket_id, bucket_id, project)
|
| 1321 |
+
print(
|
| 1322 |
+
f"* Project data uploaded to bucket: https://huggingface.co/buckets/{bucket_id}"
|
| 1323 |
+
)
|
| 1324 |
+
deploy_as_static_space(
|
| 1325 |
+
new_space_id,
|
| 1326 |
+
None,
|
| 1327 |
+
project,
|
| 1328 |
+
bucket_id=bucket_id,
|
| 1329 |
+
private=False,
|
| 1330 |
+
frontend_dir=frontend_dir,
|
| 1331 |
+
)
|
| 1332 |
+
return new_space_id
|
trackio/dummy_commit_scheduler.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from concurrent.futures import Future
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class DummyCommitSchedulerLock:
|
| 5 |
+
def __enter__(self):
|
| 6 |
+
return None
|
| 7 |
+
|
| 8 |
+
def __exit__(self, exception_type, exception_value, exception_traceback):
|
| 9 |
+
pass
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class DummyCommitScheduler:
|
| 13 |
+
def __init__(self):
|
| 14 |
+
self.lock = DummyCommitSchedulerLock()
|
| 15 |
+
|
| 16 |
+
def trigger(self) -> Future:
|
| 17 |
+
fut: Future = Future()
|
| 18 |
+
fut.set_result(None)
|
| 19 |
+
return fut
|
trackio/exceptions.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
class TrackioAPIError(Exception):
|
| 2 |
+
pass
|
trackio/fragments.py
ADDED
|
@@ -0,0 +1,389 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Append-only JSONL fragments used as a durable fallback for metric logging.
|
| 2 |
+
|
| 3 |
+
Fragments are immutable JSONL files, one writer (process) per subdirectory, so
|
| 4 |
+
concurrent training processes never contend on a shared file. They are written
|
| 5 |
+
either to a Hugging Face Bucket inbox (when a Space is unreachable) or to a
|
| 6 |
+
local inbox directory (when SQLite is unsafe, e.g. on network filesystems), and
|
| 7 |
+
are later imported into the project SQLite database by the process that owns it
|
| 8 |
+
(the Space or the dashboard server). Records carry the same ``log_id``/
|
| 9 |
+
``alert_id`` UUIDs as the HTTP logging endpoints, and imports use
|
| 10 |
+
``INSERT OR IGNORE``, so importing a fragment is idempotent: fragments are
|
| 11 |
+
deleted only after a successful import, and re-importing after a crash is
|
| 12 |
+
harmless.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import tempfile
|
| 16 |
+
import threading
|
| 17 |
+
import uuid
|
| 18 |
+
from collections.abc import Callable
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from typing import Any
|
| 21 |
+
|
| 22 |
+
import huggingface_hub
|
| 23 |
+
import orjson
|
| 24 |
+
|
| 25 |
+
from trackio import utils
|
| 26 |
+
from trackio.typehints import AlertEntry, LogEntry, SystemLogEntry
|
| 27 |
+
|
| 28 |
+
FRAGMENT_VERSION = 1
|
| 29 |
+
INBOX_DIR_NAME = "inbox"
|
| 30 |
+
BUCKET_INBOX_PREFIX = "trackio/inbox"
|
| 31 |
+
BUCKET_MEDIA_PREFIX = "trackio/media"
|
| 32 |
+
|
| 33 |
+
METRIC_KIND = "metric"
|
| 34 |
+
SYSTEM_METRIC_KIND = "system_metric"
|
| 35 |
+
ALERT_KIND = "alert"
|
| 36 |
+
KINDS = {METRIC_KIND, SYSTEM_METRIC_KIND, ALERT_KIND}
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def local_inbox_dir() -> Path:
|
| 40 |
+
return utils.TRACKIO_DIR / INBOX_DIR_NAME
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def metric_record(entry: LogEntry | dict) -> dict:
|
| 44 |
+
return {
|
| 45 |
+
"v": FRAGMENT_VERSION,
|
| 46 |
+
"kind": METRIC_KIND,
|
| 47 |
+
"project": entry["project"],
|
| 48 |
+
"run": entry["run"],
|
| 49 |
+
"run_id": entry.get("run_id"),
|
| 50 |
+
"metrics": utils.serialize_values(entry.get("metrics") or {}),
|
| 51 |
+
"step": entry.get("step"),
|
| 52 |
+
"timestamp": entry.get("timestamp"),
|
| 53 |
+
"config": utils.serialize_values(entry.get("config"))
|
| 54 |
+
if entry.get("config")
|
| 55 |
+
else None,
|
| 56 |
+
"log_id": entry.get("log_id"),
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def system_metric_record(entry: SystemLogEntry | dict) -> dict:
|
| 61 |
+
return {
|
| 62 |
+
"v": FRAGMENT_VERSION,
|
| 63 |
+
"kind": SYSTEM_METRIC_KIND,
|
| 64 |
+
"project": entry["project"],
|
| 65 |
+
"run": entry["run"],
|
| 66 |
+
"run_id": entry.get("run_id"),
|
| 67 |
+
"metrics": utils.serialize_values(entry.get("metrics") or {}),
|
| 68 |
+
"timestamp": entry.get("timestamp"),
|
| 69 |
+
"log_id": entry.get("log_id"),
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def alert_record(entry: AlertEntry | dict) -> dict:
|
| 74 |
+
return {
|
| 75 |
+
"v": FRAGMENT_VERSION,
|
| 76 |
+
"kind": ALERT_KIND,
|
| 77 |
+
"project": entry["project"],
|
| 78 |
+
"run": entry["run"],
|
| 79 |
+
"run_id": entry.get("run_id"),
|
| 80 |
+
"title": entry["title"],
|
| 81 |
+
"text": entry.get("text"),
|
| 82 |
+
"level": entry.get("level"),
|
| 83 |
+
"step": entry.get("step"),
|
| 84 |
+
"timestamp": entry.get("timestamp"),
|
| 85 |
+
"alert_id": entry.get("alert_id"),
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def bucket_media_path(
|
| 90 |
+
project: str,
|
| 91 |
+
run: str | None,
|
| 92 |
+
step: int | None,
|
| 93 |
+
relative_path: str | None,
|
| 94 |
+
filename: str,
|
| 95 |
+
) -> str:
|
| 96 |
+
parts = [BUCKET_MEDIA_PREFIX, utils.canonical_project_name(project)]
|
| 97 |
+
if run:
|
| 98 |
+
parts.append(run)
|
| 99 |
+
if step is not None:
|
| 100 |
+
parts.append(str(step))
|
| 101 |
+
else:
|
| 102 |
+
parts.append("files")
|
| 103 |
+
if relative_path:
|
| 104 |
+
parts.append(str(relative_path))
|
| 105 |
+
parts.append(filename)
|
| 106 |
+
return "/".join(parts)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
class FragmentWriter:
|
| 110 |
+
"""Writes immutable JSONL fragments for a single writer (process)."""
|
| 111 |
+
|
| 112 |
+
def __init__(self, writer_id: str | None = None):
|
| 113 |
+
self.writer_id = writer_id or uuid.uuid4().hex[:16]
|
| 114 |
+
self._seq = 0
|
| 115 |
+
self._lock = threading.Lock()
|
| 116 |
+
|
| 117 |
+
def _next_fragment_name(self) -> str:
|
| 118 |
+
with self._lock:
|
| 119 |
+
name = f"{self._seq:08d}.jsonl"
|
| 120 |
+
self._seq += 1
|
| 121 |
+
return name
|
| 122 |
+
|
| 123 |
+
@staticmethod
|
| 124 |
+
def serialize_records(records: list[dict]) -> bytes:
|
| 125 |
+
return b"".join(orjson.dumps(record) + b"\n" for record in records)
|
| 126 |
+
|
| 127 |
+
def write_local(
|
| 128 |
+
self, records: list[dict], inbox_dir: Path | None = None
|
| 129 |
+
) -> Path | None:
|
| 130 |
+
if not records:
|
| 131 |
+
return None
|
| 132 |
+
inbox = inbox_dir or local_inbox_dir()
|
| 133 |
+
writer_dir = inbox / self.writer_id
|
| 134 |
+
fragment_path = writer_dir / self._next_fragment_name()
|
| 135 |
+
data = self.serialize_records(records)
|
| 136 |
+
for attempt in range(3):
|
| 137 |
+
writer_dir.mkdir(parents=True, exist_ok=True)
|
| 138 |
+
try:
|
| 139 |
+
with tempfile.NamedTemporaryFile(
|
| 140 |
+
mode="wb", dir=writer_dir, suffix=".tmp", delete=False
|
| 141 |
+
) as tmp:
|
| 142 |
+
tmp.write(data)
|
| 143 |
+
tmp_path = Path(tmp.name)
|
| 144 |
+
tmp_path.replace(fragment_path)
|
| 145 |
+
return fragment_path
|
| 146 |
+
except FileNotFoundError:
|
| 147 |
+
if attempt == 2 or writer_dir.exists():
|
| 148 |
+
raise
|
| 149 |
+
|
| 150 |
+
def write_to_bucket(self, records: list[dict], bucket_id: str) -> str | None:
|
| 151 |
+
if not records:
|
| 152 |
+
return None
|
| 153 |
+
remote_path = (
|
| 154 |
+
f"{BUCKET_INBOX_PREFIX}/{self.writer_id}/{self._next_fragment_name()}"
|
| 155 |
+
)
|
| 156 |
+
huggingface_hub.batch_bucket_files(
|
| 157 |
+
bucket_id,
|
| 158 |
+
add=[(self.serialize_records(records), remote_path)],
|
| 159 |
+
token=huggingface_hub.utils.get_token(),
|
| 160 |
+
)
|
| 161 |
+
return remote_path
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def parse_fragment_bytes(data: bytes) -> list[dict]:
|
| 165 |
+
records = []
|
| 166 |
+
for line in data.splitlines():
|
| 167 |
+
line = line.strip()
|
| 168 |
+
if not line:
|
| 169 |
+
continue
|
| 170 |
+
try:
|
| 171 |
+
record = orjson.loads(line)
|
| 172 |
+
except orjson.JSONDecodeError:
|
| 173 |
+
continue
|
| 174 |
+
if isinstance(record, dict) and record.get("kind") in KINDS:
|
| 175 |
+
records.append(record)
|
| 176 |
+
return records
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def _group_by_run(records: list[dict]) -> dict[tuple, list[dict]]:
|
| 180 |
+
grouped: dict[tuple, list[dict]] = {}
|
| 181 |
+
for record in records:
|
| 182 |
+
key = (record.get("project"), record.get("run"), record.get("run_id"))
|
| 183 |
+
grouped.setdefault(key, []).append(record)
|
| 184 |
+
return grouped
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def import_records(records: list[dict]) -> int:
|
| 188 |
+
from trackio.sqlite_storage import SQLiteStorage # noqa: PLC0415
|
| 189 |
+
|
| 190 |
+
metric_records = [r for r in records if r.get("kind") == METRIC_KIND]
|
| 191 |
+
system_records = [r for r in records if r.get("kind") == SYSTEM_METRIC_KIND]
|
| 192 |
+
alert_records = [r for r in records if r.get("kind") == ALERT_KIND]
|
| 193 |
+
imported = 0
|
| 194 |
+
|
| 195 |
+
for (project, run, run_id), group in _group_by_run(metric_records).items():
|
| 196 |
+
if not project or not run:
|
| 197 |
+
continue
|
| 198 |
+
config = next((r["config"] for r in reversed(group) if r.get("config")), None)
|
| 199 |
+
has_timestamps = all(r.get("timestamp") for r in group)
|
| 200 |
+
SQLiteStorage.bulk_log(
|
| 201 |
+
project=project,
|
| 202 |
+
run=run,
|
| 203 |
+
run_id=run_id,
|
| 204 |
+
metrics_list=[r.get("metrics") or {} for r in group],
|
| 205 |
+
steps=[r.get("step") for r in group],
|
| 206 |
+
timestamps=[r["timestamp"] for r in group] if has_timestamps else None,
|
| 207 |
+
config=config,
|
| 208 |
+
log_ids=[r.get("log_id") for r in group],
|
| 209 |
+
)
|
| 210 |
+
imported += len(group)
|
| 211 |
+
|
| 212 |
+
for (project, run, run_id), group in _group_by_run(system_records).items():
|
| 213 |
+
if not project or not run:
|
| 214 |
+
continue
|
| 215 |
+
has_timestamps = all(r.get("timestamp") for r in group)
|
| 216 |
+
SQLiteStorage.bulk_log_system(
|
| 217 |
+
project=project,
|
| 218 |
+
run=run,
|
| 219 |
+
run_id=run_id,
|
| 220 |
+
metrics_list=[r.get("metrics") or {} for r in group],
|
| 221 |
+
timestamps=[r["timestamp"] for r in group] if has_timestamps else None,
|
| 222 |
+
log_ids=[r.get("log_id") for r in group],
|
| 223 |
+
)
|
| 224 |
+
imported += len(group)
|
| 225 |
+
|
| 226 |
+
for (project, run, run_id), group in _group_by_run(alert_records).items():
|
| 227 |
+
if not project or not run:
|
| 228 |
+
continue
|
| 229 |
+
has_timestamps = all(r.get("timestamp") for r in group)
|
| 230 |
+
SQLiteStorage.bulk_alert(
|
| 231 |
+
project=project,
|
| 232 |
+
run=run,
|
| 233 |
+
run_id=run_id,
|
| 234 |
+
titles=[r.get("title") or "" for r in group],
|
| 235 |
+
texts=[r.get("text") for r in group],
|
| 236 |
+
levels=[r.get("level") or "WARN" for r in group],
|
| 237 |
+
steps=[r.get("step") for r in group],
|
| 238 |
+
timestamps=[r["timestamp"] for r in group] if has_timestamps else None,
|
| 239 |
+
alert_ids=[r.get("alert_id") for r in group],
|
| 240 |
+
)
|
| 241 |
+
imported += len(group)
|
| 242 |
+
|
| 243 |
+
return imported
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def import_inbox_dir(inbox_dir: Path | None = None, batch_size: int = 100) -> int:
|
| 247 |
+
inbox = inbox_dir or local_inbox_dir()
|
| 248 |
+
if not inbox.exists():
|
| 249 |
+
return 0
|
| 250 |
+
imported = 0
|
| 251 |
+
batch_records: list[dict] = []
|
| 252 |
+
batch_paths: list[Path] = []
|
| 253 |
+
|
| 254 |
+
def _flush_batch() -> int:
|
| 255 |
+
nonlocal batch_records, batch_paths
|
| 256 |
+
if not batch_paths:
|
| 257 |
+
return 0
|
| 258 |
+
count = 0
|
| 259 |
+
if batch_records:
|
| 260 |
+
count = import_records(batch_records)
|
| 261 |
+
for p in batch_paths:
|
| 262 |
+
try:
|
| 263 |
+
p.unlink()
|
| 264 |
+
except OSError:
|
| 265 |
+
pass
|
| 266 |
+
batch_records = []
|
| 267 |
+
batch_paths = []
|
| 268 |
+
return count
|
| 269 |
+
|
| 270 |
+
for fragment_path in sorted(inbox.rglob("*.jsonl")):
|
| 271 |
+
try:
|
| 272 |
+
data = fragment_path.read_bytes()
|
| 273 |
+
except OSError:
|
| 274 |
+
continue
|
| 275 |
+
records = parse_fragment_bytes(data)
|
| 276 |
+
if records:
|
| 277 |
+
batch_records.extend(records)
|
| 278 |
+
batch_paths.append(fragment_path)
|
| 279 |
+
|
| 280 |
+
if len(batch_paths) >= batch_size:
|
| 281 |
+
imported += _flush_batch()
|
| 282 |
+
|
| 283 |
+
imported += _flush_batch()
|
| 284 |
+
for writer_dir in inbox.glob("*"):
|
| 285 |
+
if writer_dir.is_dir():
|
| 286 |
+
try:
|
| 287 |
+
writer_dir.rmdir()
|
| 288 |
+
except OSError:
|
| 289 |
+
pass
|
| 290 |
+
return imported
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def list_bucket_inbox_paths(bucket_id: str) -> list[str]:
|
| 294 |
+
try:
|
| 295 |
+
items = huggingface_hub.list_bucket_tree(
|
| 296 |
+
bucket_id,
|
| 297 |
+
prefix=BUCKET_INBOX_PREFIX,
|
| 298 |
+
recursive=True,
|
| 299 |
+
token=huggingface_hub.utils.get_token(),
|
| 300 |
+
)
|
| 301 |
+
except Exception:
|
| 302 |
+
return []
|
| 303 |
+
return sorted(
|
| 304 |
+
item.path
|
| 305 |
+
for item in items
|
| 306 |
+
if getattr(item, "type", None) == "file"
|
| 307 |
+
and getattr(item, "path", "").endswith(".jsonl")
|
| 308 |
+
)
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def import_inbox_from_bucket(bucket_id: str) -> int:
|
| 312 |
+
paths = list_bucket_inbox_paths(bucket_id)
|
| 313 |
+
if not paths:
|
| 314 |
+
return 0
|
| 315 |
+
imported = 0
|
| 316 |
+
consumed: list[str] = []
|
| 317 |
+
with tempfile.TemporaryDirectory() as tmp_dir:
|
| 318 |
+
for i, remote_path in enumerate(paths):
|
| 319 |
+
local_path = Path(tmp_dir) / f"{i}.jsonl"
|
| 320 |
+
try:
|
| 321 |
+
huggingface_hub.download_bucket_files(
|
| 322 |
+
bucket_id,
|
| 323 |
+
files=[(remote_path, str(local_path))],
|
| 324 |
+
token=huggingface_hub.utils.get_token(),
|
| 325 |
+
)
|
| 326 |
+
records = parse_fragment_bytes(local_path.read_bytes())
|
| 327 |
+
except Exception:
|
| 328 |
+
continue
|
| 329 |
+
if records:
|
| 330 |
+
imported += import_records(records)
|
| 331 |
+
consumed.append(remote_path)
|
| 332 |
+
if consumed:
|
| 333 |
+
try:
|
| 334 |
+
huggingface_hub.batch_bucket_files(
|
| 335 |
+
bucket_id,
|
| 336 |
+
delete=consumed,
|
| 337 |
+
token=huggingface_hub.utils.get_token(),
|
| 338 |
+
)
|
| 339 |
+
except Exception:
|
| 340 |
+
pass
|
| 341 |
+
return imported
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
def _add_files_to_bucket(bucket_id: str, additions: list[tuple[str, str]]) -> None:
|
| 345 |
+
if additions:
|
| 346 |
+
huggingface_hub.batch_bucket_files(
|
| 347 |
+
bucket_id,
|
| 348 |
+
add=additions,
|
| 349 |
+
token=huggingface_hub.utils.get_token(),
|
| 350 |
+
)
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def _upload_files_to_bucket(
|
| 354 |
+
bucket_id: str,
|
| 355 |
+
uploads: list[dict[str, Any]],
|
| 356 |
+
remote_path: Callable[[dict[str, Any], Path], str],
|
| 357 |
+
) -> None:
|
| 358 |
+
_add_files_to_bucket(
|
| 359 |
+
bucket_id,
|
| 360 |
+
[
|
| 361 |
+
(str(p), remote_path(upload, p))
|
| 362 |
+
for upload in uploads
|
| 363 |
+
if (p := Path(upload["file_path"])).exists()
|
| 364 |
+
],
|
| 365 |
+
)
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
def upload_media_files_to_bucket(bucket_id: str, uploads: list[dict[str, Any]]) -> None:
|
| 369 |
+
_upload_files_to_bucket(
|
| 370 |
+
bucket_id,
|
| 371 |
+
uploads,
|
| 372 |
+
lambda upload, p: bucket_media_path(
|
| 373 |
+
project=upload["project"],
|
| 374 |
+
run=upload.get("run"),
|
| 375 |
+
step=upload.get("step"),
|
| 376 |
+
relative_path=upload.get("relative_path"),
|
| 377 |
+
filename=p.name,
|
| 378 |
+
),
|
| 379 |
+
)
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def upload_artifact_blobs_to_bucket(
|
| 383 |
+
bucket_id: str, uploads: list[dict[str, Any]]
|
| 384 |
+
) -> None:
|
| 385 |
+
_upload_files_to_bucket(
|
| 386 |
+
bucket_id,
|
| 387 |
+
uploads,
|
| 388 |
+
lambda upload, p: f"trackio/{p.relative_to(utils.TRACKIO_DIR).as_posix()}",
|
| 389 |
+
)
|
trackio/frontend/dist/assets/EXT_lights_area-DpkQYXlQ.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{V as a,Q as R}from"./math.vector-C1xyPDEl.js";import{Color3 as L}from"./math.color-CzLaKyb6.js";import{L as c}from"./light-DT53MGnW.js";import{L as D,_ as b,s as w}from"./bufferUtils-DTRdUD8L.js";import{N as S}from"./node-C-Iu0ALa.js";import{R as M}from"./math.scalar.functions-DkeaWAWz.js";import{R as y}from"./rawTexture-Cp8_XuDs.js";import{T as d}from"./texture-BEqhDl5Y.js";import{T as A}from"./tools-D01N7ptR.js";import{T as B}from"./mesh-Fj-9677A.js";import{A as E,G as C}from"./glTFLoader-BPeiodqB.js";import{u as U,r as z}from"./dynamic-BUKysMyT.js";import"./lightConstants-B1TSTh-k.js";import"./decorators.serialization-C7d7ykGG.js";import"./baseTexture-CgAggsey.js";import"./math.plane-Ce_75-g5.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./index-Bx1QoKVU.js";import"./math.axis-Ck7JfPPj.js";import"./material-CrEsrP1R.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";import"./objectModelMapping-DWcnlpHl.js";async function F(){const i=new Uint16Array(16384),t=new Uint16Array(4096*4),r=await A.LoadFileAsync(A.GetAssetUrl("https://assets.babylonjs.com/core/areaLights/areaLightsLTC.bin")),e=new Uint16Array(r),n=e.length/8;for(let s=0;s<n;s++)i[s*4]=e[s*8],i[s*4+1]=e[s*8+1],i[s*4+2]=e[s*8+2],i[s*4+3]=e[s*8+3],t[s*4]=e[s*8+4],t[s*4+1]=e[s*8+5],t[s*4+2]=e[s*8+6],t[s*4+3]=e[s*8+7];return[i,t]}function P(i){const t=i.useDelayedTextureLoading;i.useDelayedTextureLoading=!1;const r=i._blockEntityCollection;i._blockEntityCollection=!1,i._ltcTextures={LTC1:y.CreateRGBATexture(null,64,64,i.getEngine(),!1,!1,2,2,0,!1,!0),LTC2:y.CreateRGBATexture(null,64,64,i.getEngine(),!1,!1,2,2,0,!1,!0)},i._blockEntityCollection=r,i._ltcTextures.LTC1.wrapU=d.CLAMP_ADDRESSMODE,i._ltcTextures.LTC1.wrapV=d.CLAMP_ADDRESSMODE,i._ltcTextures.LTC2.wrapU=d.CLAMP_ADDRESSMODE,i._ltcTextures.LTC2.wrapV=d.CLAMP_ADDRESSMODE,i.useDelayedTextureLoading=t,F().then(e=>{var n,s;i._ltcTextures&&(((n=i._ltcTextures)==null?void 0:n.LTC1).update(e[0]),((s=i._ltcTextures)==null?void 0:s.LTC2).update(e[1]),i.onDisposeObservable.addOnce(()=>{var o,l;(o=i._ltcTextures)==null||o.LTC1.dispose(),(l=i._ltcTextures)==null||l.LTC2.dispose()}))}).catch(e=>{D.Error(`Area Light fail to get LTC textures data. Error: ${e}`)})}class O extends c{constructor(t,r,e,n){super(t,e,n),this.position=r,this._scene._ltcTextures||P(this._scene)}transferTexturesToEffect(t,r){return this._scene._ltcTextures&&(t.setTexture("areaLightsLTC1Sampler",this._scene._ltcTextures.LTC1),t.setTexture("areaLightsLTC2Sampler",this._scene._ltcTextures.LTC2)),this}prepareLightSpecificDefines(t,r){t["AREALIGHT"+r]=!0,t.AREALIGHTUSED=!0}_isReady(){return this._scene._ltcTextures?this._scene._ltcTextures.LTC1.isReady()&&this._scene._ltcTextures.LTC2.isReady():!1}}S.AddNodeConstructor("Light_Type_4",(i,t)=>()=>new u(i,a.Zero(),1,1,t));class u extends O{get emissionTexture(){return this._emissionTextureTexture}set emissionTexture(t){this._emissionTextureTexture!==t&&(this._emissionTextureTexture=t,this._emissionTextureTexture&&(this._emissionTextureTexture.wrapU=0,this._emissionTextureTexture.wrapV=0),this._emissionTextureTexture&&u._IsTexture(this._emissionTextureTexture)&&this._emissionTextureTexture.onLoadObservable.addOnce(()=>{this._markMeshesAsLightDirty()}))}get width(){return this._width.x}set width(t){this._width.x=t}get height(){return this._height.y}set height(t){this._height.y=t}constructor(t,r,e,n,s,f){super(t,r,s,f),this._emissionTextureTexture=null,this._width=new a(e,0,0),this._height=new a(0,n,0),this._pointTransformedPosition=a.Zero(),this._pointTransformedWidth=a.Zero(),this._pointTransformedHeight=a.Zero()}getClassName(){return"RectAreaLight"}getTypeID(){return c.LIGHTTYPEID_RECT_AREALIGHT}_buildUniformLayout(){this._uniformBuffer.addUniform("vLightData",4),this._uniformBuffer.addUniform("vLightDiffuse",4),this._uniformBuffer.addUniform("vLightSpecular",4),this._uniformBuffer.addUniform("vLightWidth",4),this._uniformBuffer.addUniform("vLightHeight",4),this._uniformBuffer.addUniform("shadowsInfo",3),this._uniformBuffer.addUniform("depthValues",2),this._uniformBuffer.create()}_computeTransformedInformation(){return this.parent&&this.parent.getWorldMatrix?(a.TransformCoordinatesToRef(this.position,this.parent.getWorldMatrix(),this._pointTransformedPosition),a.TransformNormalToRef(this._width,this.parent.getWorldMatrix(),this._pointTransformedWidth),a.TransformNormalToRef(this._height,this.parent.getWorldMatrix(),this._pointTransformedHeight),!0):!1}static _IsTexture(t){return t.onLoadObservable!==void 0}transferToEffect(t,r){const e=this._scene.floatingOriginOffset;return this._computeTransformedInformation()?(this._uniformBuffer.updateFloat4("vLightData",this._pointTransformedPosition.x-e.x,this._pointTransformedPosition.y-e.y,this._pointTransformedPosition.z-e.z,0,r),this._uniformBuffer.updateFloat4("vLightWidth",this._pointTransformedWidth.x/2,this._pointTransformedWidth.y/2,this._pointTransformedWidth.z/2,0,r),this._uniformBuffer.updateFloat4("vLightHeight",this._pointTransformedHeight.x/2,this._pointTransformedHeight.y/2,this._pointTransformedHeight.z/2,0,r)):(this._uniformBuffer.updateFloat4("vLightData",this.position.x-e.x,this.position.y-e.y,this.position.z-e.z,0,r),this._uniformBuffer.updateFloat4("vLightWidth",this._width.x/2,this._width.y/2,this._width.z/2,0,r),this._uniformBuffer.updateFloat4("vLightHeight",this._height.x/2,this._height.y/2,this._height.z/2,0,r)),this}transferTexturesToEffect(t,r){return super.transferTexturesToEffect(t,r),this._emissionTextureTexture&&this._emissionTextureTexture.isReady()&&t.setTexture("rectAreaLightEmissionTexture"+r,this._emissionTextureTexture),this}transferToNodeMaterialEffect(t,r){const e=this._scene.floatingOriginOffset;return this._computeTransformedInformation()?t.setFloat3(r,this._pointTransformedPosition.x-e.x,this._pointTransformedPosition.y-e.y,this._pointTransformedPosition.z-e.z):t.setFloat3(r,this.position.x-e.x,this.position.y-e.y,this.position.z-e.z),this}prepareLightSpecificDefines(t,r){super.prepareLightSpecificDefines(t,r),t["RECTAREALIGHTEMISSIONTEXTURE"+r]=!!(this._emissionTextureTexture&&this._emissionTextureTexture.isReady())}}b([w()],u.prototype,"width",null);b([w()],u.prototype,"height",null);M("BABYLON.RectAreaLight",u);const m="EXT_lights_area";class v{constructor(t){this.name=m,this._loader=t,this.enabled=this._loader.isExtensionUsed(m)}dispose(){this._loader=null,delete this._lights}onLoading(){const t=this._loader.gltf.extensions;if(t&&t[this.name]){const r=t[this.name];this._lights=r.lights,E.Assign(this._lights)}}loadNodeAsync(t,r,e){return C.LoadExtensionAsync(t,r,this.name,async(n,s)=>(this._loader._allMaterialsDirtyRequired=!0,await this._loader.loadNodeAsync(t,r,f=>{var x;let h;const o=E.Get(n,this._lights,s.light),l=o.name||f.name;this._loader.babylonScene._blockEntityCollection=!!this._loader._assetContainer;const T=o.size!==void 0?o.size:1;switch(o.type){case"rect":{const _=((x=o.rect)==null?void 0:x.aspect)!==void 0?o.rect.aspect*T:T,g=T;h=new u(l,a.Zero(),_,g,this._loader.babylonScene);break}case"disk":{const _=Math.sqrt(T*T*.25*Math.PI);h=new u(l,a.Zero(),_,_,this._loader.babylonScene);break}default:throw this._loader.babylonScene._blockEntityCollection=!1,new Error(`${n}: Invalid area light type (${o.type})`)}h._parentContainer=this._loader._assetContainer,this._loader.babylonScene._blockEntityCollection=!1,o._babylonLight=h,h.falloffType=c.FALLOFF_GLTF,h.diffuse=o.color?L.FromArray(o.color):L.White(),h.intensity=o.intensity==null?1:o.intensity;const p=new B(`${l}_orientation`,this._loader.babylonScene);p.rotationQuaternion=R.RotationAxis(a.Up(),Math.PI),p.parent=f,h.parent=p,this._loader._babylonLights.push(h),C.AddPointerMetadata(h,n),e(f)})))}}U(m);z(m,!0,i=>new v(i));export{v as EXT_lights_area};
|
trackio/frontend/dist/assets/EXT_lights_ies-Qe4ehPsC.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{V as d}from"./math.vector-C1xyPDEl.js";import{Color3 as f}from"./math.color-CzLaKyb6.js";import{c as g}from"./objectModelMapping-DWcnlpHl.js";import{L as b}from"./light-DT53MGnW.js";import{A as l,G as h}from"./glTFLoader-BPeiodqB.js";import{u as y,r as L}from"./dynamic-BUKysMyT.js";import{T as A}from"./texture-BEqhDl5Y.js";import"./math.scalar.functions-DkeaWAWz.js";import"./bufferUtils-DTRdUD8L.js";import"./node-C-Iu0ALa.js";import"./decorators.serialization-C7d7ykGG.js";import"./math.axis-Ck7JfPPj.js";import"./lightConstants-B1TSTh-k.js";import"./tools-D01N7ptR.js";import"./index-Bx1QoKVU.js";import"./mesh-Fj-9677A.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./material-CrEsrP1R.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./baseTexture-CgAggsey.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";const a="EXT_lights_ies";class w{constructor(t){this.name=a,this._loader=t,this.enabled=this._loader.isExtensionUsed(a)}dispose(){this._loader=null,delete this._lights}onLoading(){const t=this._loader.gltf.extensions;if(t&&t[this.name]){const o=t[this.name];this._lights=o.lights,l.Assign(this._lights)}}loadNodeAsync(t,o,_){return h.LoadExtensionAsync(t,o,this.name,async(p,s)=>{this._loader._allMaterialsDirtyRequired=!0;let i,e;const u=await this._loader.loadNodeAsync(t,o,r=>{e=l.Get(p,this._lights,s.light);const c=e.name||r.name;this._loader.babylonScene._blockEntityCollection=!!this._loader._assetContainer,i=new g(c,d.Zero(),d.Backward(),0,1,this._loader.babylonScene),i.angle=Math.PI/2,i.innerAngle=0,i._parentContainer=this._loader._assetContainer,this._loader.babylonScene._blockEntityCollection=!1,e._babylonLight=i,i.falloffType=b.FALLOFF_GLTF,i.diffuse=s.color?f.FromArray(s.color):f.White(),i.intensity=s.multiplier||1,i.range=Number.MAX_VALUE,i.parent=r,this._loader._babylonLights.push(i),h.AddPointerMetadata(i,p),_(r)});let n;if(e.uri)n=await this._loader.loadUriAsync(t,e,e.uri);else{const r=l.Get(`${t}/bufferView`,this._loader.gltf.bufferViews,e.bufferView);n=await this._loader.loadBufferViewAsync(`/bufferViews/${r.index}`,r)}return i.iesProfileTexture=new A(name+"_iesProfile",this._loader.babylonScene,!0,!1,void 0,null,null,n,!0,void 0,void 0,void 0,void 0,".ies"),u})}}y(a);L(a,!0,m=>new w(m));export{w as EXT_lights_ies};
|
trackio/frontend/dist/assets/EXT_lights_image_based-CEXtWEZ3.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{a as G,S as z}from"./engine-DaBDhQ20.js";import{M as c,O as D,V as S,T as M,Q as w}from"./math.vector-C1xyPDEl.js";import{S as A}from"./decorators.serialization-C7d7ykGG.js";import{_ as F}from"./environmentTextureTools-CDrTJ7w-.js";import{_ as h,s as _,a as E,v as P}from"./bufferUtils-DTRdUD8L.js";import{T as Y}from"./tools-D01N7ptR.js";import{B as $}from"./baseTexture-CgAggsey.js";import{T as B}from"./texture-BEqhDl5Y.js";import{G as V,R as H}from"./math.scalar.functions-DkeaWAWz.js";import"./abstractEngine.cubeTexture-C895Ky-E.js";import{G as j,A as C}from"./glTFLoader-BPeiodqB.js";import{u as q,r as O}from"./dynamic-BUKysMyT.js";import"./thinEngine-D5ynu8Dd.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./dataBuffer-CgoqiUzT.js";import"./index-Bx1QoKVU.js";import"./math.axis-Ck7JfPPj.js";import"./math.color-CzLaKyb6.js";import"./math.plane-Ce_75-g5.js";import"./math.path-DFucxaHn.js";import"./perfCounter-DusmGz8t.js";import"./scene-gn_hoGMy.js";import"./smartArray-CO6iJ0u5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneComponent-Dflhczui.js";import"./buffer-DqKlo1ts.js";import"./renderingManager-CiY2OUrf.js";import"./deviceInputEvents-BTza-zoR.js";import"./passPostProcess-kvX6RZRb.js";import"./effectRenderer-CSz5fi0n.js";import"./math.viewport-CgkTt1RS.js";import"./drawWrapper-CZVk0xN-.js";import"./renderTargetTexture-B0kOC1Jy.js";import"./dumpTools-phHCiahG.js";import"./baseTexture.polynomial-BRyqQovE.js";import"./cubemapToSphericalPolynomial-BhsWOHWR.js";import"./mesh-Fj-9677A.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./subMesh-UWDo5wjI.js";import"./node-C-Iu0ALa.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./material-CrEsrP1R.js";import"./cameraInputsManager-WrTH9PVf.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";import"./light-DT53MGnW.js";import"./objectModelMapping-DWcnlpHl.js";const I=.8;class l extends ${set boundingBoxSize(e){if(this._boundingBoxSize&&this._boundingBoxSize.equals(e))return;this._boundingBoxSize=e;const t=this.getScene();t&&t.markAllMaterialsAsDirty(1)}get boundingBoxSize(){return this._boundingBoxSize}set rotationY(e){this._rotationY=e,this.setReflectionTextureMatrix(c.RotationY(this._rotationY))}get rotationY(){return this._rotationY}get noMipmap(){return this._noMipmap}get forcedExtension(){return this._forcedExtension}static CreateFromImages(e,t,i){let r="";for(const s of e)r+=s;return new l(r,t,null,i,e)}static CreateFromPrefilteredData(e,t,i=null,r=!0){const s=t.useDelayedTextureLoading;t.useDelayedTextureLoading=!1;const o=new l(e,t,null,!1,null,null,null,void 0,!0,i,r);return t.useDelayedTextureLoading=s,o}constructor(e,t,i=null,r=!1,s=null,o=null,n=null,a=5,u=!1,d=null,f=!1,x=I,p=0,y,m){var v;super(t),this.onLoadObservable=new D,this.boundingBoxPosition=S.Zero(),this._rotationY=0,this._files=null,this._forcedExtension=null,this._extensions=null,this._textureMatrixRefraction=new c,this._buffer=null,this.name=e,this.url=e,this._noMipmap=r,this.hasAlpha=!1,this.isCube=!0,this._textureMatrix=c.Identity(),this.coordinatesMode=B.CUBIC_MODE;let b,T=null;i!==null&&!Array.isArray(i)?(b=i.extensions??null,this._noMipmap=i.noMipmap??!1,s=i.files??null,T=i.buffer??null,this._format=i.format??5,u=i.prefiltered??!1,d=i.forcedExtension??null,this._createPolynomials=i.createPolynomials??!1,this._lodScale=i.lodScale??I,this._lodOffset=i.lodOffset??0,this._loaderOptions=i.loaderOptions,this._useSRGBBuffer=i.useSRGBBuffer,o=i.onLoad??null,n=i.onError??null):(this._noMipmap=r,this._format=a,this._createPolynomials=f,b=i,this._loaderOptions=y,this._useSRGBBuffer=m,this._lodScale=x,this._lodOffset=p),!(!e&&!s)&&this.updateURL(e,d,o,u,n,b,(v=this.getScene())==null?void 0:v.useDelayedTextureLoading,s,T)}getClassName(){return"CubeTexture"}updateURL(e,t=null,i=null,r=!1,s=null,o=null,n=!1,a=null,u=null){(!this.name||this.name.startsWith("data:"))&&(this.name=e),this.url=e,t&&(this._forcedExtension=t);const d=e.lastIndexOf("."),f=t||(d>-1?e.substring(d).toLowerCase():""),x=f.indexOf(".dds")===0,p=f.indexOf(".env")===0,y=f.indexOf(".basis")===0;if(p?(this.gammaSpace=!1,this._prefiltered=!1,this.anisotropicFilteringLevel=1):(this._prefiltered=r,r&&(this.gammaSpace=!1,this.anisotropicFilteringLevel=1)),a)this._files=a;else if(!y&&!p&&!x&&!o&&(o=["_px.jpg","_py.jpg","_pz.jpg","_nx.jpg","_ny.jpg","_nz.jpg"]),this._files=this._files||[],this._files.length=0,o){for(let m=0;m<o.length;m++)this._files.push(e+o[m]);this._extensions=o}this._buffer=u,n?(this.delayLoadState=4,this._delayedOnLoad=i,this._delayedOnError=s):this._loadTexture(i,s)}delayLoad(e){this.delayLoadState===4&&(e&&(this._forcedExtension=e),this.delayLoadState=1,this._loadTexture(this._delayedOnLoad,this._delayedOnError))}getReflectionTextureMatrix(){return this._textureMatrix}setReflectionTextureMatrix(e){var s,o;if(e.updateFlag===this._textureMatrix.updateFlag||(e.isIdentity()!==this._textureMatrix.isIdentity()&&((s=this.getScene())==null||s.markAllMaterialsAsDirty(1,n=>n.getActiveTextures().indexOf(this)!==-1)),this._textureMatrix=e,!((o=this.getScene())!=null&&o.useRightHandedSystem)))return;const t=M.Vector3[0],i=M.Quaternion[0],r=M.Vector3[1];this._textureMatrix.decompose(t,i,r),i.z*=-1,i.w*=-1,c.ComposeToRef(t,i,r,this._textureMatrixRefraction)}getRefractionTextureMatrix(){var e;return(e=this.getScene())!=null&&e.useRightHandedSystem?this._textureMatrixRefraction:this._textureMatrix}_loadTexture(e=null,t=null){var n;const i=this.getScene(),r=this._texture;this._texture=this._getFromCache(this.url,this._noMipmap,void 0,void 0,this._useSRGBBuffer,this.isCube);const s=()=>{var a;this.onLoadObservable.notifyObservers(this),r&&(r.dispose(),(a=this.getScene())==null||a.markAllMaterialsAsDirty(1)),e&&e()},o=(a,u)=>{this._loadingError=!0,this._errorObject={message:a,exception:u},t&&t(a,u),B.OnTextureLoadErrorObservable.notifyObservers(this)};this._texture?this._texture.isReady?Y.SetImmediate(()=>s()):this._texture.onLoadedObservable.add(()=>s()):(this._prefiltered?this._texture=this._getEngine().createPrefilteredCubeTexture(this.url,i,this._lodScale,this._lodOffset,e,o,this._format,this._forcedExtension,this._createPolynomials):this._texture=this._getEngine().createCubeTexture(this.url,i,this._files,this._noMipmap,e,o,this._format,this._forcedExtension,!1,this._lodScale,this._lodOffset,null,this._loaderOptions,!!this._useSRGBBuffer,this._buffer),(n=this._texture)==null||n.onLoadedObservable.add(()=>this.onLoadObservable.notifyObservers(this)))}static Parse(e,t,i){const r=A.Parse(()=>{let s=!1;return e.prefiltered&&(s=e.prefiltered),new l(i+(e.url??e.name),t,e.extensions,!1,e.files||null,null,null,void 0,s,e.forcedExtension)},e,t);if(e.boundingBoxPosition&&(r.boundingBoxPosition=S.FromArray(e.boundingBoxPosition)),e.boundingBoxSize&&(r.boundingBoxSize=S.FromArray(e.boundingBoxSize)),e.animations)for(let s=0;s<e.animations.length;s++){const o=e.animations[s],n=V("BABYLON.Animation");n&&r.animations.push(n.Parse(o))}return r}clone(){let e=0;const t=A.Clone(()=>{const i=new l(this.url,this.getScene()||this._getEngine(),this._extensions,this._noMipmap,this._files);return e=i.uniqueId,i},this);return t.uniqueId=e,t}}h([_()],l.prototype,"url",void 0);h([E()],l.prototype,"boundingBoxPosition",void 0);h([E()],l.prototype,"boundingBoxSize",null);h([_("rotationY")],l.prototype,"rotationY",null);h([_("files")],l.prototype,"_files",void 0);h([_("forcedExtension")],l.prototype,"_forcedExtension",void 0);h([_("extensions")],l.prototype,"_extensions",void 0);h([P("textureMatrix")],l.prototype,"_textureMatrix",void 0);h([P("textureMatrixRefraction")],l.prototype,"_textureMatrixRefraction",void 0);B._CubeTextureParser=l.Parse;H("BABYLON.CubeTexture",l);class R extends l{constructor(e,t,i,r=5,s=0,o=!1,n=!1,a=3,u=null){super("",e),this._texture=e.getEngine().createRawCubeTexture(t,i,r,s,o,n,a,u)}update(e,t,i,r,s=null){this._texture.getEngine().updateRawCubeTexture(this._texture,e,t,i,r,s)}updateRGBDAsync(e,t=null,i=.8,r=0){return F(this._texture,e,t,i,r).then(()=>{})}clone(){return A.Clone(()=>{const e=this.getScene(),t=this._texture,i=new R(e,t._bufferViewArray,t.width,t.format,t.type,t.generateMipMaps,t.invertY,t.samplingMode,t._compression);return t.source===13&&i.updateRGBDAsync(t._bufferViewArrayArray,t._sphericalPolynomial,t._lodGenerationScale,t._lodGenerationOffset),i},this)}}const g="EXT_lights_image_based";class N{constructor(e){this.name=g,this._loader=e,this.enabled=this._loader.isExtensionUsed(g)}dispose(){this._loader=null,delete this._lights}onLoading(){const e=this._loader.gltf.extensions;if(e&&e[this.name]){const t=e[this.name];this._lights=t.lights}}loadSceneAsync(e,t){return j.LoadExtensionAsync(e,t,this.name,async(i,r)=>{this._loader._allMaterialsDirtyRequired=!0;const s=new Array;s.push(this._loader.loadSceneAsync(e,t)),this._loader.logOpen(`${i}`);const o=C.Get(`${i}/light`,this._lights,r.light);return s.push(this._loadLightAsync(`/extensions/${this.name}/lights/${r.light}`,o).then(n=>{this._loader.babylonScene.environmentTexture=n})),this._loader.logClose(),await Promise.all(s).then(()=>{})})}_loadLightAsync(e,t){if(!t._loaded){const i=new Array;this._loader.logOpen(`${e}`);const r=new Array(t.specularImages.length);for(let s=0;s<t.specularImages.length;s++){const o=t.specularImages[s];r[s]=new Array(o.length);for(let n=0;n<o.length;n++){const a=`${e}/specularImages/${s}/${n}`;this._loader.logOpen(`${a}`);const u=o[n],d=C.Get(a,this._loader.gltf.images,u);i.push(this._loader.loadImageAsync(`/images/${u}`,d).then(f=>{r[s][n]=f})),this._loader.logClose()}}this._loader.logClose(),t._loaded=Promise.all(i).then(async()=>{const s=new R(this._loader.babylonScene,null,t.specularImageSize);if(s.name=t.name||"environment",t._babylonTexture=s,t.intensity!=null&&(s.level=t.intensity),t.rotation){let u=w.FromArray(t.rotation);this._loader.babylonScene.useRightHandedSystem||(u=w.Inverse(u)),c.FromQuaternionToRef(u,s.getReflectionTextureMatrix())}if(!t.irradianceCoefficients)throw new Error(`${e}: Irradiance coefficients are missing`);const o=G.FromArray(t.irradianceCoefficients);o.scaleInPlace(t.intensity),o.convertIrradianceToLambertianRadiance();const n=z.FromHarmonics(o),a=(r.length-1)/Math.log2(t.specularImageSize);return await s.updateRGBDAsync(r,n,a)})}return t._loaded.then(()=>t._babylonTexture)}}q(g);O(g,!0,L=>new N(L));export{N as EXT_lights_image_based};
|
trackio/frontend/dist/assets/EXT_mesh_gpu_instancing-B3VxVLJg.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{T as t,V as A,Q as F,M as b}from"./math.vector-C1xyPDEl.js";import{L as g}from"./bufferUtils-DTRdUD8L.js";import{G as L,A as M}from"./glTFLoader-BPeiodqB.js";import{u as V,r as w}from"./dynamic-BUKysMyT.js";import"./thinInstanceMesh-CYnaUZKe.js";import"./math.scalar.functions-DkeaWAWz.js";import"./index-Bx1QoKVU.js";import"./math.color-CzLaKyb6.js";import"./tools-D01N7ptR.js";import"./decorators.serialization-C7d7ykGG.js";import"./mesh-Fj-9677A.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./node-C-Iu0ALa.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./math.axis-Ck7JfPPj.js";import"./material-CrEsrP1R.js";import"./texture-BEqhDl5Y.js";import"./baseTexture-CgAggsey.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";import"./light-DT53MGnW.js";import"./objectModelMapping-DWcnlpHl.js";const m="EXT_mesh_gpu_instancing";class B{constructor(p){this.name=m,this._loader=p,this.enabled=this._loader.isExtensionUsed(m)}dispose(){this._loader=null}loadNodeAsync(p,i,T){return L.LoadExtensionAsync(p,i,this.name,async(f,u)=>{this._loader._disableInstancedMesh++;const h=this._loader.loadNodeAsync(`/nodes/${i.index}`,i,T);if(this._loader._disableInstancedMesh--,!i._primitiveBabylonMeshes)return await h;const c=new Array;let o=0;const n=s=>{if(u.attributes[s]==null){c.push(Promise.resolve(null));return}const e=M.Get(`${f}/attributes/${s}`,this._loader.gltf.accessors,u.attributes[s]);if(c.push(this._loader._loadFloatAccessorAsync(`/accessors/${e.bufferView}`,e)),o===0)o=e.count;else if(o!==e.count)throw new Error(`${f}/attributes: Instance buffer accessors do not have the same count.`)};return n("TRANSLATION"),n("ROTATION"),n("SCALE"),n("_COLOR_0"),await h.then(async s=>{const[e,d,_,a]=await Promise.all(c),y=new Float32Array(o*16);t.Vector3[0].copyFromFloats(0,0,0),t.Quaternion[0].copyFromFloats(0,0,0,1),t.Vector3[1].copyFromFloats(1,1,1);for(let r=0;r<o;++r)e&&A.FromArrayToRef(e,r*3,t.Vector3[0]),d&&F.FromArrayToRef(d,r*4,t.Quaternion[0]),_&&A.FromArrayToRef(_,r*3,t.Vector3[1]),b.ComposeToRef(t.Vector3[1],t.Quaternion[0],t.Vector3[0],t.Matrix[0]),t.Matrix[0].copyToArray(y,r*16);for(const r of i._primitiveBabylonMeshes)r.thinInstanceSetBuffer("matrix",y,16,!0),a&&(a.length===o*3?r.thinInstanceSetBuffer("color",a,3,!0):a.length===o*4?r.thinInstanceSetBuffer("color",a,4,!0):g.Warn("Unexpected size of _COLOR_0 attribute for mesh "+r.name));return s})})}}V(m);w(m,!0,l=>new B(l));export{B as EXT_mesh_gpu_instancing};
|
trackio/frontend/dist/assets/EXT_meshopt_compression-Pa499AYZ.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{G as n,A as c}from"./glTFLoader-BPeiodqB.js";import{T as f}from"./tools-D01N7ptR.js";import{u as h,r as y}from"./dynamic-BUKysMyT.js";import"./index-Bx1QoKVU.js";import"./math.vector-C1xyPDEl.js";import"./math.scalar.functions-DkeaWAWz.js";import"./math.color-CzLaKyb6.js";import"./mesh-Fj-9677A.js";import"./decorators.serialization-C7d7ykGG.js";import"./bufferUtils-DTRdUD8L.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./node-C-Iu0ALa.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./math.axis-Ck7JfPPj.js";import"./material-CrEsrP1R.js";import"./texture-BEqhDl5Y.js";import"./baseTexture-CgAggsey.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";import"./light-DT53MGnW.js";import"./objectModelMapping-DWcnlpHl.js";let p=0,m=null;class e{static get Default(){return e._Default||(e._Default=new e),e._Default}constructor(){const t=e.Configuration.decoder;this._decoderModulePromise=f.LoadBabylonScriptAsync(t.url).then(()=>MeshoptDecoder.ready)}dispose(){delete this._decoderModulePromise}async decodeGltfBufferAsync(t,i,d,r,o){await this._decoderModulePromise,p===0&&(MeshoptDecoder.useWorkers(1),p=1);const s=await MeshoptDecoder.decodeGltfBufferAsync(i,d,t,r,o);return m!==null&&clearTimeout(m),m=setTimeout(()=>{MeshoptDecoder.useWorkers(0),p=0,m=null},1e3),s}}e.Configuration={decoder:{url:`${f._DefaultCdnUrl}/meshopt_decoder.js`}};e._Default=null;const a="EXT_meshopt_compression";class _{constructor(t){this.name=a,this.enabled=t.isExtensionUsed(a),this._loader=t}dispose(){this._loader=null}loadBufferViewAsync(t,i){return n.LoadExtensionAsync(t,i,this.name,async(d,r)=>{const o=i;if(o._meshOptData)return await o._meshOptData;const s=c.Get(`${t}/buffer`,this._loader.gltf.buffers,r.buffer);return o._meshOptData=this._loader.loadBufferAsync(`/buffers/${s.index}`,s,r.byteOffset||0,r.byteLength).then(async l=>await e.Default.decodeGltfBufferAsync(l,r.count,r.byteStride,r.mode,r.filter)),await o._meshOptData})}}h(a);y(a,!0,u=>new _(u));export{_ as EXT_meshopt_compression};
|
trackio/frontend/dist/assets/EXT_texture_avif-Nv7VBjPQ.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{G as m,A as p}from"./glTFLoader-BPeiodqB.js";import{u,r as _}from"./dynamic-BUKysMyT.js";import"./index-Bx1QoKVU.js";import"./math.vector-C1xyPDEl.js";import"./math.scalar.functions-DkeaWAWz.js";import"./math.color-CzLaKyb6.js";import"./tools-D01N7ptR.js";import"./decorators.serialization-C7d7ykGG.js";import"./bufferUtils-DTRdUD8L.js";import"./mesh-Fj-9677A.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./node-C-Iu0ALa.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./math.axis-Ck7JfPPj.js";import"./material-CrEsrP1R.js";import"./texture-BEqhDl5Y.js";import"./baseTexture-CgAggsey.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";import"./light-DT53MGnW.js";import"./objectModelMapping-DWcnlpHl.js";const o="EXT_texture_avif";class c{constructor(r){this.name=o,this._loader=r,this.enabled=r.isExtensionUsed(o)}dispose(){this._loader=null}_loadTextureAsync(r,t,e){return m.LoadExtensionAsync(r,t,this.name,async(s,a)=>{const n=t.sampler==null?m.DefaultSampler:p.Get(`${r}/sampler`,this._loader.gltf.samplers,t.sampler),l=p.Get(`${s}/source`,this._loader.gltf.images,a.source);return await this._loader._createTextureAsync(r,n,l,d=>{e(d)},void 0,!t._textureInfo.nonColorData)})}}u(o);_(o,!0,i=>new c(i));export{c as EXT_texture_avif};
|
trackio/frontend/dist/assets/EXT_texture_webp-BDYrjNRo.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{G as m,A as p}from"./glTFLoader-BPeiodqB.js";import{u,r as _}from"./dynamic-BUKysMyT.js";import"./index-Bx1QoKVU.js";import"./math.vector-C1xyPDEl.js";import"./math.scalar.functions-DkeaWAWz.js";import"./math.color-CzLaKyb6.js";import"./tools-D01N7ptR.js";import"./decorators.serialization-C7d7ykGG.js";import"./bufferUtils-DTRdUD8L.js";import"./mesh-Fj-9677A.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./node-C-Iu0ALa.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./math.axis-Ck7JfPPj.js";import"./material-CrEsrP1R.js";import"./texture-BEqhDl5Y.js";import"./baseTexture-CgAggsey.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";import"./light-DT53MGnW.js";import"./objectModelMapping-DWcnlpHl.js";const o="EXT_texture_webp";class c{constructor(r){this.name=o,this._loader=r,this.enabled=r.isExtensionUsed(o)}dispose(){this._loader=null}_loadTextureAsync(r,t,e){return m.LoadExtensionAsync(r,t,this.name,async(s,a)=>{const n=t.sampler==null?m.DefaultSampler:p.Get(`${r}/sampler`,this._loader.gltf.samplers,t.sampler),l=p.Get(`${s}/source`,this._loader.gltf.images,a.source);return await this._loader._createTextureAsync(r,n,l,d=>{e(d)},void 0,!t._textureInfo.nonColorData)})}}u(o);_(o,!0,i=>new c(i));export{c as EXT_texture_webp};
|
trackio/frontend/dist/assets/ExtrasAsMetadata-Cp9P1Lmx.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{u as o,r as n}from"./dynamic-BUKysMyT.js";import"./index-Bx1QoKVU.js";import"./sceneLoader-CutT-qCn.js";import"./tools-D01N7ptR.js";import"./math.vector-C1xyPDEl.js";import"./math.scalar.functions-DkeaWAWz.js";import"./decorators.serialization-C7d7ykGG.js";import"./bufferUtils-DTRdUD8L.js";import"./math.color-CzLaKyb6.js";import"./scene-gn_hoGMy.js";import"./smartArray-CO6iJ0u5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./renderingManager-CiY2OUrf.js";import"./deviceInputEvents-BTza-zoR.js";import"./perfCounter-DusmGz8t.js";import"./sceneLoaderFlags-wywyb9ml.js";const i="ExtrasAsMetadata";class m{_assignExtras(r,t){if(t.extras&&Object.keys(t.extras).length>0){const s=r.metadata=r.metadata||{},a=s.gltf=s.gltf||{};a.extras=t.extras}}constructor(r){this.name=i,this.enabled=!0,this._loader=r}dispose(){this._loader=null}loadNodeAsync(r,t,s){return this._loader.loadNodeAsync(r,t,a=>{this._assignExtras(a,t),s(a)})}loadCameraAsync(r,t,s){return this._loader.loadCameraAsync(r,t,a=>{this._assignExtras(a,t),s(a)})}createMaterial(r,t,s){const a=this._loader.createMaterial(r,t,s);return this._assignExtras(a,t),a}loadAnimationAsync(r,t){return this._loader.loadAnimationAsync(r,t).then(s=>(this._assignExtras(s,t),s))}}o(i);n(i,!1,e=>new m(e));export{m as ExtrasAsMetadata};
|
trackio/frontend/dist/assets/KHR_animation_pointer-DbEjDTwi.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{L as K}from"./bufferUtils-DTRdUD8L.js";import{u as B,r as Z}from"./dynamic-BUKysMyT.js";import{S as e,a as G}from"./objectModelMapping-DWcnlpHl.js";import{A as t}from"./animation-IftAdLIx.js";import{AnimationPropertyInfo as p}from"./glTFLoaderAnimation-QPpM8WOy.js";import{Color3 as V}from"./math.color-CzLaKyb6.js";import"./index-Bx1QoKVU.js";import"./sceneLoader-CutT-qCn.js";import"./tools-D01N7ptR.js";import"./math.vector-C1xyPDEl.js";import"./math.scalar.functions-DkeaWAWz.js";import"./decorators.serialization-C7d7ykGG.js";import"./scene-gn_hoGMy.js";import"./smartArray-CO6iJ0u5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./renderingManager-CiY2OUrf.js";import"./deviceInputEvents-BTza-zoR.js";import"./perfCounter-DusmGz8t.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./node-C-Iu0ALa.js";import"./light-DT53MGnW.js";import"./math.axis-Ck7JfPPj.js";import"./texture-BEqhDl5Y.js";import"./baseTexture-CgAggsey.js";function u(l,r,a,i){return V.FromArray(r,a).scale(i)}function W(l,r,a,i){return r[a+3]*i}function s(l,r,a,i){return r[a]*i}function O(l,r,a,i){return-r[a]*i}function A(l,r,a,i){return r[a+1]*i}function z(l,r,a,i){return r[a]*i*2}function o(l){return{scale:[new n(t.ANIMATIONTYPE_FLOAT,`${l}.uScale`,s,()=>2),new n(t.ANIMATIONTYPE_FLOAT,`${l}.vScale`,A,()=>2)],offset:[new n(t.ANIMATIONTYPE_FLOAT,`${l}.uOffset`,s,()=>2),new n(t.ANIMATIONTYPE_FLOAT,`${l}.vOffset`,A,()=>2)],rotation:[new n(t.ANIMATIONTYPE_FLOAT,`${l}.wAng`,O,()=>1)]}}class c extends p{buildAnimations(r,a,i,m){return[{babylonAnimatable:r._babylonCamera,babylonAnimation:this._buildAnimation(a,i,m)}]}}class n extends p{buildAnimations(r,a,i,m){const x=[];for(const T in r._data)x.push({babylonAnimatable:r._data[T].babylonMaterial,babylonAnimation:this._buildAnimation(a,i,m)});return x}}class _ extends p{buildAnimations(r,a,i,m){return[{babylonAnimatable:r._babylonLight,babylonAnimation:this._buildAnimation(a,i,m)}]}}class D extends p{buildAnimations(r,a,i,m){return r._primitiveBabylonMeshes?r._primitiveBabylonMeshes.map(x=>({babylonAnimatable:x,babylonAnimation:this._buildAnimation(a,i,m)})):[]}}e("/cameras/{}/orthographic/xmag",[new c(t.ANIMATIONTYPE_FLOAT,"orthoLeft",O,()=>1),new c(t.ANIMATIONTYPE_FLOAT,"orthoRight",A,()=>1)]);e("/cameras/{}/orthographic/ymag",[new c(t.ANIMATIONTYPE_FLOAT,"orthoBottom",O,()=>1),new c(t.ANIMATIONTYPE_FLOAT,"orthoTop",A,()=>1)]);e("/cameras/{}/orthographic/zfar",[new c(t.ANIMATIONTYPE_FLOAT,"maxZ",s,()=>1)]);e("/cameras/{}/orthographic/znear",[new c(t.ANIMATIONTYPE_FLOAT,"minZ",s,()=>1)]);e("/cameras/{}/perspective/yfov",[new c(t.ANIMATIONTYPE_FLOAT,"fov",s,()=>1)]);e("/cameras/{}/perspective/zfar",[new c(t.ANIMATIONTYPE_FLOAT,"maxZ",s,()=>1)]);e("/cameras/{}/perspective/znear",[new c(t.ANIMATIONTYPE_FLOAT,"minZ",s,()=>1)]);e("/materials/{}/pbrMetallicRoughness/baseColorFactor",[new n(t.ANIMATIONTYPE_COLOR3,"albedoColor",u,()=>4),new n(t.ANIMATIONTYPE_FLOAT,"alpha",W,()=>4)]);e("/materials/{}/pbrMetallicRoughness/metallicFactor",[new n(t.ANIMATIONTYPE_FLOAT,"metallic",s,()=>1)]);e("/materials/{}/pbrMetallicRoughness/metallicFactor",[new n(t.ANIMATIONTYPE_FLOAT,"roughness",s,()=>1)]);const H=o("albedoTexture");e("/materials/{}/pbrMetallicRoughness/baseColorTexture/extensions/KHR_texture_transform/scale",H.scale);e("/materials/{}/pbrMetallicRoughness/baseColorTexture/extensions/KHR_texture_transform/offset",H.offset);e("/materials/{}/pbrMetallicRoughness/baseColorTexture/extensions/KHR_texture_transform/rotation",H.rotation);const N=o("metallicTexture");e("//materials/{}/pbrMetallicRoughness/metallicRoughnessTexture/scale",N.scale);e("//materials/{}/pbrMetallicRoughness/metallicRoughnessTexture/offset",N.offset);e("//materials/{}/pbrMetallicRoughness/metallicRoughnessTexture/rotation",N.rotation);e("/materials/{}/emissiveFactor",[new n(t.ANIMATIONTYPE_COLOR3,"emissiveColor",u,()=>3)]);const g=o("bumpTexture");e("/materials/{}/normalTexture/scale",[new n(t.ANIMATIONTYPE_FLOAT,"bumpTexture.level",s,()=>1)]);e("/materials/{}/normalTexture/extensions/KHR_texture_transform/scale",g.scale);e("/materials/{}/normalTexture/extensions/KHR_texture_transform/offset",g.offset);e("/materials/{}/normalTexture/extensions/KHR_texture_transform/rotation",g.rotation);e("/materials/{}/occlusionTexture/strength",[new n(t.ANIMATIONTYPE_FLOAT,"ambientTextureStrength",s,()=>1)]);const d=o("ambientTexture");e("/materials/{}/occlusionTexture/extensions/KHR_texture_transform/scale",d.scale);e("/materials/{}/occlusionTexture/extensions/KHR_texture_transform/offset",d.offset);e("/materials/{}/occlusionTexture/extensions/KHR_texture_transform/rotation",d.rotation);const b=o("emissiveTexture");e("/materials/{}/emissiveTexture/extensions/KHR_texture_transform/scale",b.scale);e("/materials/{}/emissiveTexture/extensions/KHR_texture_transform/offset",b.offset);e("/materials/{}/emissiveTexture/extensions/KHR_texture_transform/rotation",b.rotation);e("/materials/{}/extensions/KHR_materials_anisotropy/anisotropyStrength",[new n(t.ANIMATIONTYPE_FLOAT,"anisotropy.intensity",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_anisotropy/anisotropyRotation",[new n(t.ANIMATIONTYPE_FLOAT,"anisotropy.angle",s,()=>1)]);const M=o("anisotropy.texture");e("/materials/{}/extensions/KHR_materials_anisotropy/anisotropyTexture/extensions/KHR_texture_transform/scale",M.scale);e("/materials/{}/extensions/KHR_materials_anisotropy/anisotropyTexture/extensions/KHR_texture_transform/offset",M.offset);e("/materials/{}/extensions/KHR_materials_anisotropy/anisotropyTexture/extensions/KHR_texture_transform/rotation",M.rotation);e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatFactor",[new n(t.ANIMATIONTYPE_FLOAT,"clearCoat.intensity",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatRoughnessFactor",[new n(t.ANIMATIONTYPE_FLOAT,"clearCoat.roughness",s,()=>1)]);const F=o("clearCoat.texture");e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatTexture/extensions/KHR_texture_transform/scale",F.scale);e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatTexture/extensions/KHR_texture_transform/offset",F.offset);e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatTexture/extensions/KHR_texture_transform/rotation",F.rotation);const E=o("clearCoat.bumpTexture");e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatNormalTexture/scale",[new n(t.ANIMATIONTYPE_FLOAT,"clearCoat.bumpTexture.level",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatNormalTexture/extensions/KHR_texture_transform/scale",E.scale);e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatNormalTexture/extensions/KHR_texture_transform/offset",E.offset);e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatNormalTexture/extensions/KHR_texture_transform/rotation",E.rotation);const w=o("clearCoat.textureRoughness");e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatRoughnessTexture/extensions/KHR_texture_transform/scale",w.scale);e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatRoughnessTexture/extensions/KHR_texture_transform/offset",w.offset);e("/materials/{}/extensions/KHR_materials_clearcoat/clearcoatRoughnessTexture/extensions/KHR_texture_transform/rotation",w.rotation);e("/materials/{}/extensions/KHR_materials_dispersion/dispersionFactor",[new n(t.ANIMATIONTYPE_FLOAT,"subSurface.dispersion",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_emissive_strength/emissiveStrength",[new n(t.ANIMATIONTYPE_FLOAT,"emissiveIntensity",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_ior/ior",[new n(t.ANIMATIONTYPE_FLOAT,"indexOfRefraction",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_iridescence/iridescenceFactor",[new n(t.ANIMATIONTYPE_FLOAT,"iridescence.intensity",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_iridescence/iridescenceIor",[new n(t.ANIMATIONTYPE_FLOAT,"iridescence.indexOfRefraction",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_iridescence/iridescenceThicknessMinimum",[new n(t.ANIMATIONTYPE_FLOAT,"iridescence.minimumThickness",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_iridescence/iridescenceThicknessMaximum",[new n(t.ANIMATIONTYPE_FLOAT,"iridescence.maximumThickness",s,()=>1)]);const L=o("iridescence.texture");e("/materials/{}/extensions/KHR_materials_iridescence/iridescenceTexture/extensions/KHR_texture_transform/scale",L.scale);e("/materials/{}/extensions/KHR_materials_iridescence/iridescenceTexture/extensions/KHR_texture_transform/offset",L.offset);e("/materials/{}/extensions/KHR_materials_iridescence/iridescenceTexture/extensions/KHR_texture_transform/rotation",L.rotation);const P=o("iridescence.thicknessTexture");e("/materials/{}/extensions/KHR_materials_iridescence/iridescenceThicknessTexture/extensions/KHR_texture_transform/scale",P.scale);e("/materials/{}/extensions/KHR_materials_iridescence/iridescenceThicknessTexture/extensions/KHR_texture_transform/offset",P.offset);e("/materials/{}/extensions/KHR_materials_iridescence/iridescenceThicknessTexture/extensions/KHR_texture_transform/rotation",P.rotation);e("/materials/{}/extensions/KHR_materials_sheen/sheenColorFactor",[new n(t.ANIMATIONTYPE_COLOR3,"sheen.color",u,()=>3)]);e("/materials/{}/extensions/KHR_materials_sheen/sheenRoughnessFactor",[new n(t.ANIMATIONTYPE_FLOAT,"sheen.roughness",s,()=>1)]);const y=o("sheen.texture");e("/materials/{}/extensions/KHR_materials_sheen/sheenColorTexture/extensions/KHR_texture_transform/scale",y.scale);e("/materials/{}/extensions/KHR_materials_sheen/sheenColorTexture/extensions/KHR_texture_transform/offset",y.offset);e("/materials/{}/extensions/KHR_materials_sheen/sheenColorTexture/extensions/KHR_texture_transform/rotation",y.rotation);const C=o("sheen.textureRoughness");e("/materials/{}/extensions/KHR_materials_sheen/sheenRoughnessTexture/extensions/KHR_texture_transform/scale",C.scale);e("/materials/{}/extensions/KHR_materials_sheen/sheenRoughnessTexture/extensions/KHR_texture_transform/offset",C.offset);e("/materials/{}/extensions/KHR_materials_sheen/sheenRoughnessTexture/extensions/KHR_texture_transform/rotation",C.rotation);e("/materials/{}/extensions/KHR_materials_specular/specularFactor",[new n(t.ANIMATIONTYPE_FLOAT,"metallicF0Factor",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_specular/specularColorFactor",[new n(t.ANIMATIONTYPE_COLOR3,"metallicReflectanceColor",u,()=>3)]);const Y=o("metallicReflectanceTexture");e("/materials/{}/extensions/KHR_materials_specular/specularTexture/extensions/KHR_texture_transform/scale",Y.scale);e("/materials/{}/extensions/KHR_materials_specular/specularTexture/extensions/KHR_texture_transform/offset",Y.offset);e("/materials/{}/extensions/KHR_materials_specular/specularTexture/extensions/KHR_texture_transform/rotation",Y.rotation);const v=o("reflectanceTexture");e("/materials/{}/extensions/KHR_materials_specular/specularColorTexture/extensions/KHR_texture_transform/scale",v.scale);e("/materials/{}/extensions/KHR_materials_specular/specularColorTexture/extensions/KHR_texture_transform/offset",v.offset);e("/materials/{}/extensions/KHR_materials_specular/specularColorTexture/extensions/KHR_texture_transform/rotation",v.rotation);e("/materials/{}/extensions/KHR_materials_transmission/transmissionFactor",[new n(t.ANIMATIONTYPE_FLOAT,"subSurface.refractionIntensity",s,()=>1)]);const S=o("subSurface.refractionIntensityTexture");e("/materials/{}/extensions/KHR_materials_transmission/transmissionTexture/extensions/KHR_texture_transform/scale",S.scale);e("/materials/{}/extensions/KHR_materials_transmission/transmissionTexture/extensions/KHR_texture_transform/offset",S.offset);e("/materials/{}/extensions/KHR_materials_transmission/transmissionTexture/extensions/KHR_texture_transform/rotation",S.rotation);e("/materials/{}/extensions/KHR_materials_volume/attenuationColor",[new n(t.ANIMATIONTYPE_COLOR3,"subSurface.tintColor",u,()=>3)]);e("/materials/{}/extensions/KHR_materials_volume/attenuationDistance",[new n(t.ANIMATIONTYPE_FLOAT,"subSurface.tintColorAtDistance",s,()=>1)]);e("/materials/{}/extensions/KHR_materials_volume/thicknessFactor",[new n(t.ANIMATIONTYPE_FLOAT,"subSurface.maximumThickness",s,()=>1)]);const $=o("subSurface.thicknessTexture");e("/materials/{}/extensions/KHR_materials_volume/thicknessTexture/extensions/KHR_texture_transform/scale",$.scale);e("/materials/{}/extensions/KHR_materials_volume/thicknessTexture/extensions/KHR_texture_transform/offset",$.offset);e("/materials/{}/extensions/KHR_materials_volume/thicknessTexture/extensions/KHR_texture_transform/rotation",$.rotation);e("/materials/{}/extensions/KHR_materials_diffuse_transmission/diffuseTransmissionFactor",[new n(t.ANIMATIONTYPE_FLOAT,"subSurface.translucencyIntensity",s,()=>1)]);const k=o("subSurface.translucencyIntensityTexture");e("materials/{}/extensions/KHR_materials_diffuse_transmission/diffuseTransmissionTexture/extensions/KHR_texture_transform/scale",k.scale);e("materials/{}/extensions/KHR_materials_diffuse_transmission/diffuseTransmissionTexture/extensions/KHR_texture_transform/offset",k.offset);e("materials/{}/extensions/KHR_materials_diffuse_transmission/diffuseTransmissionTexture/extensions/KHR_texture_transform/rotation",k.rotation);e("/materials/{}/extensions/KHR_materials_diffuse_transmission/diffuseTransmissionColorFactor",[new n(t.ANIMATIONTYPE_COLOR3,"subSurface.translucencyColor",u,()=>3)]);const j=o("subSurface.translucencyColorTexture");e("materials/{}/extensions/KHR_materials_diffuse_transmission/diffuseTransmissionColorTexture/extensions/KHR_texture_transform/scale",j.scale);e("materials/{}/extensions/KHR_materials_diffuse_transmission/diffuseTransmissionColorTexture/extensions/KHR_texture_transform/offset",j.offset);e("materials/{}/extensions/KHR_materials_diffuse_transmission/diffuseTransmissionColorTexture/extensions/KHR_texture_transform/rotation",j.rotation);e("/extensions/KHR_lights_punctual/lights/{}/color",[new _(t.ANIMATIONTYPE_COLOR3,"diffuse",u,()=>3)]);e("/extensions/KHR_lights_punctual/lights/{}/intensity",[new _(t.ANIMATIONTYPE_FLOAT,"intensity",s,()=>1)]);e("/extensions/KHR_lights_punctual/lights/{}/range",[new _(t.ANIMATIONTYPE_FLOAT,"range",s,()=>1)]);e("/extensions/KHR_lights_punctual/lights/{}/spot/innerConeAngle",[new _(t.ANIMATIONTYPE_FLOAT,"innerAngle",z,()=>1)]);e("/extensions/KHR_lights_punctual/lights/{}/spot/outerConeAngle",[new _(t.ANIMATIONTYPE_FLOAT,"angle",z,()=>1)]);e("/extensions/EXT_lights_area/lights/{}/color",[new _(t.ANIMATIONTYPE_COLOR3,"diffuse",u,()=>3)]);e("/extensions/EXT_lights_area/lights/{}/intensity",[new _(t.ANIMATIONTYPE_FLOAT,"intensity",s,()=>1)]);e("/extensions/EXT_lights_area/lights/{}/size",[new _(t.ANIMATIONTYPE_FLOAT,"radius",s,()=>1)]);e("/extensions/EXT_lights_area/lights/{}/rect/aspect",[new _(t.ANIMATIONTYPE_FLOAT,"radius",s,()=>1)]);e("/nodes/{}/extensions/EXT_lights_ies/color",[new _(t.ANIMATIONTYPE_COLOR3,"diffuse",u,()=>3)]);e("/nodes/{}/extensions/EXT_lights_ies/multiplier",[new _(t.ANIMATIONTYPE_FLOAT,"intensity",s,()=>1)]);e("/nodes/{}/extensions/KHR_node_visibility/visible",[new D(t.ANIMATIONTYPE_FLOAT,"isVisible",s,()=>1)]);const R="KHR_animation_pointer";class U{constructor(r){this.name=R,this._loader=r,this._pathToObjectConverter=G(this._loader.gltf)}get enabled(){return this._loader.isExtensionUsed(R)}dispose(){this._loader=null,delete this._pathToObjectConverter}_loadAnimationChannelAsync(r,a,i,m,x){var X;const T=(X=m.target.extensions)==null?void 0:X.KHR_animation_pointer;if(!T||!this._pathToObjectConverter)return null;m.target.path!=="pointer"&&K.Warn(`${r}/target/path: Value (${m.target.path}) must be (pointer) when using the ${this.name} extension`),m.target.node!=null&&K.Warn(`${r}/target/node: Value (${m.target.node}) must not be present when using the ${this.name} extension`);const I=`${r}/extensions/${this.name}`,h=T.pointer;if(!h)throw new Error(`${I}: Pointer is missing`);try{const f=this._pathToObjectConverter.convert(h);if(!f.info.interpolation)throw new Error(`${I}/pointer: Interpolation is missing`);return this._loader._loadAnimationChannelFromTargetInfoAsync(r,a,i,m,{object:f.object,info:f.info.interpolation},x)}catch{return K.Warn(`${I}/pointer: Invalid pointer (${h}) skipped`),null}}}B(R);Z(R,!0,l=>new U(l));export{U as KHR_animation_pointer};
|
trackio/frontend/dist/assets/KHR_draco_mesh_compression-BEx33bus.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{T as A}from"./tools-D01N7ptR.js";import{A as v}from"./workerPool-CfMXSLnf.js";import{G as C}from"./mesh-Fj-9677A.js";import{V as _}from"./buffer-DqKlo1ts.js";import{L as V}from"./bufferUtils-DTRdUD8L.js";import{G as k,A as g,L as R}from"./glTFLoader-BPeiodqB.js";import{u as F,r as N}from"./dynamic-BUKysMyT.js";import"./math.vector-C1xyPDEl.js";import"./math.scalar.functions-DkeaWAWz.js";import"./decorators.serialization-C7d7ykGG.js";import"./math.color-CzLaKyb6.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./node-C-Iu0ALa.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./index-Bx1QoKVU.js";import"./math.axis-Ck7JfPPj.js";import"./material-CrEsrP1R.js";import"./texture-BEqhDl5Y.js";import"./baseTexture-CgAggsey.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./dataBuffer-CgoqiUzT.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";import"./light-DT53MGnW.js";import"./objectModelMapping-DWcnlpHl.js";function U(c,t,n,a,m){const e=c;let r=null,o=null,s=null;try{r=new e.Decoder,o=new e.DecoderBuffer,o.Init(t,t.byteLength);let i;const l=r.GetEncodedGeometryType(o);switch(l){case e.TRIANGULAR_MESH:{const d=new e.Mesh;if(i=r.DecodeBufferToMesh(o,d),!i.ok()||d.ptr===0)throw new Error(i.error_msg());const y=d.num_faces()*3,p=y*4,u=e._malloc(p);try{r.GetTrianglesUInt32Array(d,p,u);const P=new Uint32Array(y);P.set(new Uint32Array(e.HEAPF32.buffer,u,y)),a(P)}finally{e._free(u)}s=d;break}case e.POINT_CLOUD:{const d=new e.PointCloud;if(i=r.DecodeBufferToPointCloud(o,d),!i.ok()||!d.ptr)throw new Error(i.error_msg());s=d;break}default:throw new Error(`Draco: Cannot decode invalid geometry type ${l}`)}const h=s.num_points(),b=(d,f,y,p)=>{const u=p.data_type(),P=p.num_components(),M=p.normalized(),B=p.byte_stride(),O=p.byte_offset(),T={[e.DT_FLOAT32]:{typedArrayConstructor:Float32Array,heap:e.HEAPF32},[e.DT_INT8]:{typedArrayConstructor:Int8Array,heap:e.HEAP8},[e.DT_INT16]:{typedArrayConstructor:Int16Array,heap:e.HEAP16},[e.DT_INT32]:{typedArrayConstructor:Int32Array,heap:e.HEAP32},[e.DT_UINT8]:{typedArrayConstructor:Uint8Array,heap:e.HEAPU8},[e.DT_UINT16]:{typedArrayConstructor:Uint16Array,heap:e.HEAPU16},[e.DT_UINT32]:{typedArrayConstructor:Uint32Array,heap:e.HEAPU32}}[u];if(!T)throw new Error(`Draco: Cannot decode invalid data type ${u}`);const I=h*P,L=I*T.typedArrayConstructor.BYTES_PER_ELEMENT,D=e._malloc(L);try{d.GetAttributeDataArrayForAllPoints(f,p,u,L,D);const G=new T.typedArrayConstructor(T.heap.buffer,D,I);m(y,G.slice(),P,O,B,M)}finally{e._free(D)}};if(n)for(const d in n){const f=n[d],y=r.GetAttributeByUniqueId(s,f);b(r,s,d,y)}else{const d={position:e.POSITION,normal:e.NORMAL,color:e.COLOR,uv:e.TEX_COORD};for(const f in d){const y=r.GetAttributeId(s,d[f]);if(y!==-1){const p=r.GetAttribute(s,y);b(r,s,f,p)}}}return h}finally{s&&e.destroy(s),o&&e.destroy(o),r&&e.destroy(r)}}function S(){let c;onmessage=t=>{const n=t.data;switch(n.id){case"init":{n.url&&importScripts(n.url);const a=n.wasmBinary?{wasmBinary:n.wasmBinary}:{};c=DracoDecoderModule(a),postMessage({id:"initDone"});break}case"decodeMesh":{if(!c)throw new Error("Draco: Decoder module is not available");c.then(a=>{const m=U(a,n.dataView,n.attributes,e=>{postMessage({id:"indices",data:e},[e.buffer])},(e,r,o,s,i,l)=>{postMessage({id:"attribute",kind:e,data:r,size:o,byteOffset:s,byteStride:i,normalized:l},[r.buffer])});postMessage({id:"decodeMeshDone",totalVertices:m})});break}}}}async function W(c,t,n){return await new Promise((a,m)=>{const e=o=>{c.removeEventListener("error",e),c.removeEventListener("message",r),m(o)},r=o=>{o.data.id==="initDone"&&(c.removeEventListener("error",e),c.removeEventListener("message",r),a(c))};if(c.addEventListener("error",e),c.addEventListener("message",r),!t)c.postMessage({id:"init",url:n});else{const o=t.slice(0);c.postMessage({id:"init",url:n,wasmBinary:o},[o])}})}function K(){return typeof navigator!="object"||!navigator.hardwareConcurrency?1:Math.min(Math.floor(navigator.hardwareConcurrency*.5),4)}function $(c){return!!(c.wasmUrl&&(c.wasmBinary||c.wasmBinaryUrl)&&typeof WebAssembly=="object"||c.fallbackUrl)}class z{constructor(t){if(t.workerPool){this._workerPoolPromise=Promise.resolve(t.workerPool);return}const n=t.wasmBinary,a=t.numWorkers??K(),m=a&&typeof Worker=="function"&&typeof URL=="function",e=m||!t.jsModule,r=t.wasmUrl&&t.wasmBinaryUrl&&typeof WebAssembly=="object"?{url:e?A.GetBabylonScriptURL(t.wasmUrl,!0):"",wasmBinaryPromise:n?Promise.resolve(n):A.LoadFileAsync(A.GetBabylonScriptURL(t.wasmBinaryUrl,!0))}:{url:e?A.GetBabylonScriptURL(t.fallbackUrl):"",wasmBinaryPromise:Promise.resolve(void 0)};m?this._workerPoolPromise=r.wasmBinaryPromise.then(o=>{const s=this._getWorkerContent(),i=URL.createObjectURL(new Blob([s],{type:"application/javascript"}));return new v(a,()=>{const l=new Worker(i);return W(l,o,r.url)})}):this._modulePromise=r.wasmBinaryPromise.then(async o=>{if(!this._isModuleAvailable()&&!t.jsModule){if(!r.url)throw new Error("Draco codec module is not available");await A.LoadBabylonScriptAsync(r.url)}return await this._createModuleAsync(o,t.jsModule)})}async whenReadyAsync(){if(this._workerPoolPromise){await this._workerPoolPromise;return}if(this._modulePromise){await this._modulePromise;return}}dispose(){this._workerPoolPromise&&this._workerPoolPromise.then(t=>{t.dispose()}),delete this._workerPoolPromise,delete this._modulePromise}}class w extends z{static get DefaultAvailable(){return $(w.DefaultConfiguration)}static get Default(){return w._Default??(w._Default=new w),w._Default}static ResetDefault(t){w._Default&&(t||w._Default.dispose(),w._Default=null)}_isModuleAvailable(){return typeof DracoDecoderModule<"u"}async _createModuleAsync(t,n){return{module:await(n||DracoDecoderModule)({wasmBinary:t})}}_getWorkerContent(){return`${U}(${S})()`}constructor(t=w.DefaultConfiguration){super(t)}decodeMeshToMeshDataAsync(t,n,a){const m=t instanceof ArrayBuffer?new Int8Array(t):new Int8Array(t.buffer,t.byteOffset,t.byteLength),e=(r,o)=>a&&a[r]!==void 0?(o!==a[r]&&V.Warn(`Normalized flag from Draco data (${o}) does not match normalized flag from glTF accessor (${a[r]}). Using flag from glTF accessor.`),a[r]):o;if(this._workerPoolPromise)return this._workerPoolPromise.then(async r=>await new Promise((o,s)=>{r.push((i,l)=>{let h=null;const b=[],d=p=>{i.removeEventListener("error",d),i.removeEventListener("message",f),s(p),l()},f=p=>{const u=p.data;switch(u.id){case"indices":{h=u.data;break}case"attribute":{b.push({kind:u.kind,data:u.data,size:u.size,byteOffset:u.byteOffset,byteStride:u.byteStride,normalized:e(u.kind,u.normalized)});break}case"decodeMeshDone":{i.removeEventListener("error",d),i.removeEventListener("message",f),o({indices:h,attributes:b,totalVertices:u.totalVertices}),l();break}}};i.addEventListener("error",d),i.addEventListener("message",f);const y=m.slice();i.postMessage({id:"decodeMesh",dataView:y,attributes:n},[y.buffer])})}));if(this._modulePromise)return this._modulePromise.then(r=>{let o=null;const s=[],i=U(r.module,m,n,l=>{o=l},(l,h,b,d,f,y)=>{s.push({kind:l,data:h,size:b,byteOffset:d,byteStride:f,normalized:y})});return{indices:o,attributes:s,totalVertices:i}});throw new Error("Draco: Decoder module is not available")}async decodeMeshToGeometryAsync(t,n,a,m){const e=await this.decodeMeshToMeshDataAsync(a,m),r=new C(t,n);e.indices&&r.setIndices(e.indices);for(const o of e.attributes)r.setVerticesBuffer(new _(n.getEngine(),o.data,o.kind,!1,void 0,o.byteStride,void 0,o.byteOffset,o.size,void 0,o.normalized,!0),e.totalVertices);return r}async _decodeMeshToGeometryForGltfAsync(t,n,a,m,e,r){const o=await this.decodeMeshToMeshDataAsync(a,m,e),s=new C(t,n);r&&(s._boundingInfo=r,s.useBoundingInfoFromGeometry=!0),o.indices&&s.setIndices(o.indices);for(const i of o.attributes)s.setVerticesBuffer(new _(n.getEngine(),i.data,i.kind,!1,void 0,i.byteStride,void 0,i.byteOffset,i.size,void 0,i.normalized,!0),o.totalVertices);return s}}w.DefaultConfiguration={wasmUrl:`${A._DefaultCdnUrl}/draco_wasm_wrapper_gltf.js`,wasmBinaryUrl:`${A._DefaultCdnUrl}/draco_decoder_gltf.wasm`,fallbackUrl:`${A._DefaultCdnUrl}/draco_decoder_gltf.js`};w._Default=null;const E="KHR_draco_mesh_compression";class H{constructor(t){this.name=E,this.useNormalizedFlagFromAccessor=!0,this._loader=t,this.enabled=w.DefaultAvailable&&this._loader.isExtensionUsed(E)}dispose(){delete this.dracoDecoder,this._loader=null}_loadVertexDataAsync(t,n,a){return k.LoadExtensionAsync(t,n,this.name,async(m,e)=>{if(n.mode!=null&&n.mode!==4&&n.mode!==5)throw new Error(`${t}: Unsupported mode ${n.mode}`);const r={},o={},s=(l,h)=>{const b=e.attributes[l];if(b!=null&&(a._delayInfo=a._delayInfo||[],a._delayInfo.indexOf(h)===-1&&a._delayInfo.push(h),r[h]=b,this.useNormalizedFlagFromAccessor)){const d=g.TryGet(this._loader.gltf.accessors,n.attributes[l]);d&&(o[h]=d.normalized||!1)}};s("POSITION",_.PositionKind),s("NORMAL",_.NormalKind),s("TANGENT",_.TangentKind),s("TEXCOORD_0",_.UVKind),s("TEXCOORD_1",_.UV2Kind),s("TEXCOORD_2",_.UV3Kind),s("TEXCOORD_3",_.UV4Kind),s("TEXCOORD_4",_.UV5Kind),s("TEXCOORD_5",_.UV6Kind),s("JOINTS_0",_.MatricesIndicesKind),s("WEIGHTS_0",_.MatricesWeightsKind),s("COLOR_0",_.ColorKind);const i=g.Get(m,this._loader.gltf.bufferViews,e.bufferView);return i._dracoBabylonGeometry||(i._dracoBabylonGeometry=this._loader.loadBufferViewAsync(`/bufferViews/${i.index}`,i).then(async l=>{const h=this.dracoDecoder||w.Default,b=g.TryGet(this._loader.gltf.accessors,n.attributes.POSITION),d=!this._loader.parent.alwaysComputeBoundingBox&&!a.skeleton&&b?R(b):null;return await h._decodeMeshToGeometryForGltfAsync(a.name,this._loader.babylonScene,l,r,o,d).catch(f=>{throw new Error(`${t}: ${f.message}`)})})),await i._dracoBabylonGeometry})}}F(E);N(E,!0,c=>new H(c));export{H as KHR_draco_mesh_compression};
|
trackio/frontend/dist/assets/KHR_interactivity-K8Kioa6L.js
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
const __vite__mapDeps=(i,m=__vite__mapDeps,d=(m.f||(m.f=["assets/flowGraphPlayAnimationBlock-B9FP0RfS.js","assets/declarationMapper-oEUQrCmY.js","assets/bufferUtils-DTRdUD8L.js","assets/math.vector-C1xyPDEl.js","assets/math.scalar.functions-DkeaWAWz.js","assets/math.color-CzLaKyb6.js","assets/animationGroup-BP7VB8mY.js","assets/animation-IftAdLIx.js","assets/node-C-Iu0ALa.js","assets/decorators.serialization-C7d7ykGG.js","assets/baseTexture-CgAggsey.js","assets/bone-B35odufe.js","assets/scene-gn_hoGMy.js","assets/tools-D01N7ptR.js","assets/smartArray-CO6iJ0u5.js","assets/lightConstants-B1TSTh-k.js","assets/sceneComponent-Dflhczui.js","assets/math.plane-Ce_75-g5.js","assets/buffer-DqKlo1ts.js","assets/dataBuffer-CgoqiUzT.js","assets/renderingManager-CiY2OUrf.js","assets/deviceInputEvents-BTza-zoR.js","assets/perfCounter-DusmGz8t.js","assets/index-Bx1QoKVU.js","assets/index-BfQ3YfM4.css","assets/dynamic-BUKysMyT.js","assets/sceneLoader-CutT-qCn.js","assets/sceneLoaderFlags-wywyb9ml.js","assets/objectModelMapping-DWcnlpHl.js","assets/light-DT53MGnW.js","assets/math.axis-Ck7JfPPj.js","assets/texture-BEqhDl5Y.js","assets/flowGraphStopAnimationBlock-ChXcK4cb.js","assets/flowGraphPauseAnimationBlock-DAoY0eJc.js","assets/flowGraphInterpolationBlock-CYcMKCGO.js","assets/flowGraphSceneReadyEventBlock-Crn-rAIL.js","assets/flowGraphSceneTickEventBlock-gf3YDJRe.js","assets/flowGraphSendCustomEventBlock-Ba8XUM8G.js","assets/flowGraphReceiveCustomEventBlock-CId8KX4K.js","assets/flowGraphMeshPickEventBlock-WsLHiXtY.js","assets/flowGraphMathBlocks-HeZhspxf.js","assets/flowGraphBinaryOperationBlock-CNTF2Ejc.js","assets/flowGraphCachedOperationBlock-cxKp5Gdq.js","assets/flowGraphUnaryOperationBlock-CI7NJ04I.js","assets/flowGraphVectorMathBlocks-BGV5dKz9.js","assets/flowGraphMatrixMathBlocks-Dg7VLPFP.js","assets/flowGraphBranchBlock-CPQ1Lj8X.js","assets/flowGraphSetDelayBlock-BA13Fleb.js","assets/flowGraphCancelDelayBlock-CiPuCgGz.js","assets/flowGraphCounterBlock-OjaFZjW5.js","assets/flowGraphDebounceBlock-fxe8WoxT.js","assets/flowGraphThrottleBlock-B-yO7rgE.js","assets/flowGraphDoNBlock-C4vVyovK.js","assets/flowGraphFlipFlopBlock-BSjrwcqq.js","assets/flowGraphForLoopBlock-jI9z_K5X.js","assets/flowGraphMultiGateBlock-Cy-tdSZ3.js","assets/flowGraphSequenceBlock-Bcye5mDB.js","assets/flowGraphSwitchBlock-CfuObCnO.js","assets/flowGraphWaitAllBlock-VXxeVnO_.js","assets/flowGraphWhileLoopBlock-BYmEfSWz.js","assets/flowGraphConsoleLogBlock-Cz7IRQyV.js","assets/flowGraphConditionalDataBlock-Btzamqcx.js","assets/flowGraphConstantBlock-BV1ij11S.js","assets/flowGraphTransformCoordinatesSystemBlock-D3zEJ4A-.js","assets/flowGraphGetAssetBlock-BeMIiO3t.js","assets/flowGraphGetPropertyBlock-CAOmAIk8.js","assets/flowGraphSetPropertyBlock-DxJBj-a9.js","assets/flowGraphGetVariableBlock-BaRkHNJ_.js","assets/flowGraphSetVariableBlock-BG0Kfa6g.js","assets/flowGraphJsonPointerParserBlock-CZ93bSRG.js","assets/flowGraphMathCombineExtractBlocks-Dg02KNQM.js","assets/flowGraphTypeToTypeBlocks-DdLy9OXZ.js","assets/flowGraphEasingBlock-C_-3ZhLh.js","assets/easing-Wc0jPSIm.js","assets/math.path-DFucxaHn.js","assets/flowGraphBezierCurveEasingBlock--1XIKCm2.js","assets/flowGraphPointerOverEventBlock-wKWlgMPO.js","assets/flowGraphPointerOutEventBlock-BpW1yJKn.js","assets/flowGraphContextBlock-DeEwvFEY.js","assets/flowGraphArrayIndexBlock-3XRt4h8L.js","assets/flowGraphCodeExecutionBlock-BpjYYwgf.js","assets/flowGraphIndexOfBlock-D1XW4kc4.js","assets/flowGraphFunctionReferenceBlock-CLhl7P_Q.js","assets/flowGraphDataSwitchBlock-BUqG5JEg.js","assets/flowGraphGLTFDataProvider-By82y__G.js"])))=>i.map(i=>d[i]);
|
| 2 |
+
import{_ as r}from"./index-Bx1QoKVU.js";import{M as kt,a as Gt,V as ut,b as Bt,Q as X,O as U}from"./math.vector-C1xyPDEl.js";import{L as w,_ as Ft,s as Et}from"./bufferUtils-DTRdUD8L.js";import{R as D}from"./decorators.serialization-C7d7ykGG.js";import{Color3 as vt,Color4 as It}from"./math.color-CzLaKyb6.js";import{F as Tt,a as Vt,b as Ct,g as Pt,c as bt,d as at}from"./declarationMapper-oEUQrCmY.js";import{R as ht}from"./math.scalar.functions-DkeaWAWz.js";import{P as rt}from"./deviceInputEvents-BTza-zoR.js";import{u as Ot,r as xt}from"./dynamic-BUKysMyT.js";import{a as Dt,A as O}from"./objectModelMapping-DWcnlpHl.js";function At(a){return a==="Mesh"||a==="AbstractMesh"||a==="GroundMesh"||a==="InstanceMesh"||a==="LinesMesh"||a==="GoldbergMesh"||a==="GreasedLineMesh"||a==="TrailMesh"}function pt(a){return a==="Vector2"||a==="Vector3"||a==="Vector4"||a==="Quaternion"||a==="Color3"||a==="Color4"}function Rt(a){return a==="Matrix"||a==="Matrix2D"||a==="Matrix3D"}function Lt(a){return a==="AnimationGroup"}function St(a,t,e=!1){if(a==="Vector2")return Gt.FromArray(t);if(a==="Vector3")return e&&(t[2]*=-1),ut.FromArray(t);if(a==="Vector4")return Bt.FromArray(t);if(a==="Quaternion")return e&&(t[2]*=-1,t[3]*=-1),X.FromArray(t);if(a==="Color3")return new vt(t[0],t[1],t[2]);if(a==="Color4")return new It(t[0],t[1],t[2],t[3]);throw new Error(`Unknown vector class name ${a}`)}function tt(a,t,e){var n;const o=((n=t==null?void 0:t.getClassName)==null?void 0:n.call(t))??"";if(pt(o)||Rt(o))e[a]={value:t.asArray(),className:o};else if(o==="FlowGraphInteger")e[a]={value:t.value,className:o};else if(o&&(t.id||t.name))e[a]={id:t.id,name:t.name,className:o};else if(typeof t!="object")e[a]=t;else throw new Error(`Could not serialize value ${t}`)}function H(a,t,e,o){const n=t[a];let s;const i=(n==null?void 0:n.type)??(n==null?void 0:n.className);if(At(i)){let c=o.meshes.filter(l=>n.id?l.id===n.id:l.name===n.name);c.length===0&&(c=o.transformNodes.filter(l=>n.id?l.id===n.id:l.name===n.name)),s=n.uniqueId?c.find(l=>l.uniqueId===n.uniqueId):c[0]}else if(pt(i))s=St(i,n.value);else if(Lt(i)){const c=o.animationGroups.filter(l=>l.name===n.name);s=c.length===1?c[0]:c.find(l=>l.uniqueId===n.uniqueId)}else i==="Matrix"?s=kt.FromArray(n.value):i==="Matrix2D"?s=new Tt(n.value):i==="Matrix3D"?s=new Vt(n.value):i==="FlowGraphInteger"?s=Ct.FromValue(n.value):i==="number"||i==="string"||i==="boolean"?s=n.value[0]:n&&n.value!==void 0?s=n.value:Array.isArray(n)?s=n.reduce((c,l)=>(l.eventData&&(c[l.id]={type:Pt(l.type)},typeof l.value<"u"&&(c[l.id].value=H("value",l,e,o))),c),{}):s=n;return s}function Nt(a){return a==="FlowGraphJsonPointerParserBlock"}var st;(function(a){a.Animation="Animation",a.AnimationGroup="AnimationGroup",a.Mesh="Mesh",a.Material="Material",a.Camera="Camera",a.Light="Light"})(st||(st={}));function Mt(a,t,e,o){switch(t){case"Animation":return o?a.animations.find(n=>n.uniqueId===e)??null:a.animations[e]??null;case"AnimationGroup":return o?a.animationGroups.find(n=>n.uniqueId===e)??null:a.animationGroups[e]??null;case"Mesh":return o?a.meshes.find(n=>n.uniqueId===e)??null:a.meshes[e]??null;case"Material":return o?a.materials.find(n=>n.uniqueId===e)??null:a.materials[e]??null;case"Camera":return o?a.cameras.find(n=>n.uniqueId===e)??null:a.cameras[e]??null;case"Light":return o?a.lights.find(n=>n.uniqueId===e)??null:a.lights[e]??null;default:return null}}var it;(function(a){a.ExecuteBlock="ExecuteBlock",a.ExecuteEvent="ExecuteEvent",a.TriggerConnection="TriggerConnection",a.ContextVariableSet="ContextVariableSet",a.GlobalVariableSet="GlobalVariableSet",a.GlobalVariableDelete="GlobalVariableDelete",a.GlobalVariableGet="GlobalVariableGet",a.AddConnection="AddConnection",a.GetConnectionValue="GetConnectionValue",a.SetConnectionValue="SetConnectionValue",a.ActivateSignal="ActivateSignal",a.ContextVariableGet="ContextVariableGet"})(it||(it={}));class qt{constructor(){this.logToConsole=!1,this.log=[]}addLogItem(t){var e;if(t.time||(t.time=Date.now()),this.log.push(t),this.logToConsole){const o=(e=t.payload)==null?void 0:e.value;typeof o=="object"&&o.getClassName?w.Log(`[FGLog] ${t.className}:${t.uniqueId.split("-")[0]} ${t.action} - ${JSON.stringify(o.getClassName())}: ${o.toString()}`):w.Log(`[FGLog] ${t.className}:${t.uniqueId.split("-")[0]} ${t.action} - ${JSON.stringify(t.payload)}`)}}getItemsOfType(t){return this.log.filter(e=>e.action===t)}}class _t{get enableLogging(){return this._enableLogging}set enableLogging(t){this._enableLogging!==t&&(this._enableLogging=t,this._enableLogging?(this.logger=new qt,this.logger.logToConsole=!0):this.logger=null)}constructor(t){this.uniqueId=D(),this._userVariables={},this._executionVariables={},this._globalContextVariables={},this._connectionValues={},this._pendingBlocks=[],this._executionId=0,this.onNodeExecutedObservable=new U,this.treatDataAsRightHanded=!1,this._enableLogging=!1,this._configuration=t,this.assetsContext=t.assetsContext??t.scene}hasVariable(t){return t in this._userVariables}setVariable(t,e){var o;this._userVariables[t]=e,(o=this.logger)==null||o.addLogItem({time:Date.now(),className:this.getClassName(),uniqueId:this.uniqueId,action:"ContextVariableSet",payload:{name:t,value:e}})}getAsset(t,e){return Mt(this.assetsContext,t,e)}getVariable(t){var e;return(e=this.logger)==null||e.addLogItem({time:Date.now(),className:this.getClassName(),uniqueId:this.uniqueId,action:"ContextVariableGet",payload:{name:t,value:this._userVariables[t]}}),this._userVariables[t]}get userVariables(){return this._userVariables}getScene(){return this._configuration.scene}_getUniqueIdPrefixedName(t,e){return`${t.uniqueId}_${e}`}_getGlobalContextVariable(t,e){var o;return(o=this.logger)==null||o.addLogItem({time:Date.now(),className:this.getClassName(),uniqueId:this.uniqueId,action:"GlobalVariableGet",payload:{name:t,defaultValue:e,possibleValue:this._globalContextVariables[t]}}),this._hasGlobalContextVariable(t)?this._globalContextVariables[t]:e}_setGlobalContextVariable(t,e){var o;(o=this.logger)==null||o.addLogItem({time:Date.now(),className:this.getClassName(),uniqueId:this.uniqueId,action:"GlobalVariableSet",payload:{name:t,value:e}}),this._globalContextVariables[t]=e}_deleteGlobalContextVariable(t){var e;(e=this.logger)==null||e.addLogItem({time:Date.now(),className:this.getClassName(),uniqueId:this.uniqueId,action:"GlobalVariableDelete",payload:{name:t}}),delete this._globalContextVariables[t]}_hasGlobalContextVariable(t){return t in this._globalContextVariables}_setExecutionVariable(t,e,o){this._executionVariables[this._getUniqueIdPrefixedName(t,e)]=o}_getExecutionVariable(t,e,o){return this._hasExecutionVariable(t,e)?this._executionVariables[this._getUniqueIdPrefixedName(t,e)]:o}_deleteExecutionVariable(t,e){delete this._executionVariables[this._getUniqueIdPrefixedName(t,e)]}_hasExecutionVariable(t,e){return this._getUniqueIdPrefixedName(t,e)in this._executionVariables}_hasConnectionValue(t){return t.uniqueId in this._connectionValues}_setConnectionValue(t,e){var o;this._connectionValues[t.uniqueId]=e,(o=this.logger)==null||o.addLogItem({time:Date.now(),className:this.getClassName(),uniqueId:this.uniqueId,action:"SetConnectionValue",payload:{connectionPointId:t.uniqueId,value:e}})}_setConnectionValueByKey(t,e){this._connectionValues[t]=e}_getConnectionValue(t){var e;return(e=this.logger)==null||e.addLogItem({time:Date.now(),className:this.getClassName(),uniqueId:this.uniqueId,action:"GetConnectionValue",payload:{connectionPointId:t.uniqueId,value:this._connectionValues[t.uniqueId]}}),this._connectionValues[t.uniqueId]}get configuration(){return this._configuration}get hasPendingBlocks(){return this._pendingBlocks.length>0}_addPendingBlock(t){this._pendingBlocks.includes(t)||(this._pendingBlocks.push(t),this._pendingBlocks.sort((e,o)=>e.priority-o.priority))}_removePendingBlock(t){const e=this._pendingBlocks.indexOf(t);e!==-1&&this._pendingBlocks.splice(e,1)}_clearPendingBlocks(){for(const t of this._pendingBlocks)t._cancelPendingTasks(this);this._pendingBlocks.length=0}_notifyExecuteNode(t){var e;this.onNodeExecutedObservable.notifyObservers(t),(e=this.logger)==null||e.addLogItem({time:Date.now(),className:t.getClassName(),uniqueId:t.uniqueId,action:"ExecuteBlock"})}_notifyOnTick(t){var e;this._setGlobalContextVariable("timeSinceStart",t.timeSinceStart),this._setGlobalContextVariable("deltaTime",t.deltaTime);for(const o of this._pendingBlocks)(e=o._executeOnTick)==null||e.call(o,this)}_increaseExecutionId(){this._executionId++}get executionId(){return this._executionId}serialize(t={},e=tt){var o;t.uniqueId=this.uniqueId,t._userVariables={};for(const n in this._userVariables)e(n,this._userVariables[n],t._userVariables);t._connectionValues={};for(const n in this._connectionValues)e(n,this._connectionValues[n],t._connectionValues);this.assetsContext!==this.getScene()&&(t._assetsContext={meshes:this.assetsContext.meshes.map(n=>n.id),materials:this.assetsContext.materials.map(n=>n.id),textures:this.assetsContext.textures.map(n=>n.name),animations:this.assetsContext.animations.map(n=>n.name),lights:this.assetsContext.lights.map(n=>n.id),cameras:this.assetsContext.cameras.map(n=>n.id),sounds:(o=this.assetsContext.sounds)==null?void 0:o.map(n=>n.name),skeletons:this.assetsContext.skeletons.map(n=>n.id),particleSystems:this.assetsContext.particleSystems.map(n=>n.name),geometries:this.assetsContext.geometries.map(n=>n.id),multiMaterials:this.assetsContext.multiMaterials.map(n=>n.id),transformNodes:this.assetsContext.transformNodes.map(n=>n.id)})}getClassName(){return"FlowGraphContext"}}Ft([Et()],_t.prototype,"uniqueId",void 0);var ct;(function(a){a[a.Input=0]="Input",a[a.Output=1]="Output"})(ct||(ct={}));class wt{constructor(t,e,o){this._ownerBlock=o,this._connectedPoint=[],this.uniqueId=D(),this.connectedPointIds=[],this.name=t,this._connectionType=e}get connectionType(){return this._connectionType}_isSingularConnection(){return!0}isConnected(){return this._connectedPoint.length>0}connectTo(t){if(this._connectionType===t._connectionType)throw new Error(`Cannot connect two points of type ${this.connectionType}`);if(this._isSingularConnection()&&this._connectedPoint.length>0||t._isSingularConnection()&&t._connectedPoint.length>0)throw new Error("Max number of connections for point reached");this._connectedPoint.push(t),t._connectedPoint.push(this)}disconnectFrom(t,e=!0){const o=this._connectedPoint.indexOf(t),n=t._connectedPoint.indexOf(this);o===-1||n===-1||(e&&this._connectedPoint.splice(o,1),t._connectedPoint.splice(n,1))}disconnectFromAll(){for(const t of this._connectedPoint)this.disconnectFrom(t,!1);this._connectedPoint.length=0}dispose(){for(const t of this._connectedPoint)this.disconnectFrom(t)}serialize(t={}){t.uniqueId=this.uniqueId,t.name=this.name,t._connectionType=this._connectionType,t.connectedPointIds=[],t.className=this.getClassName();for(const e of this._connectedPoint)t.connectedPointIds.push(e.uniqueId)}getClassName(){return"FGConnection"}deserialize(t){this.uniqueId=t.uniqueId,this.name=t.name,this._connectionType=t._connectionType,this.connectedPointIds=t.connectedPointIds}}class Y extends wt{constructor(t,e,o,n,s=n.defaultValue,i=!1){super(t,e,o),this.richType=n,this._defaultValue=s,this._optional=i,this._isDisabled=!1,this._lastValue=null,this.dataTransformer=null,this.onValueChangedObservable=new U}get optional(){return this._optional}get isDisabled(){return this._isDisabled}set isDisabled(t){this._isDisabled!==t&&(this._isDisabled=t,this._isDisabled&&this.disconnectFromAll())}_isSingularConnection(){return this.connectionType===0}setValue(t,e){e._getConnectionValue(this)!==t&&(e._setConnectionValue(this,t),this.onValueChangedObservable.notifyObservers(t))}resetToDefaultValue(t){t._setConnectionValue(this,this._defaultValue)}connectTo(t){this._isDisabled||super.connectTo(t)}_getValueOrDefault(t){const e=t._getConnectionValue(this)??this._defaultValue;return this.dataTransformer?this.dataTransformer(e):e}getValue(t){if(this.connectionType===1){t._notifyExecuteNode(this._ownerBlock),this._ownerBlock._updateOutputs(t);const o=this._getValueOrDefault(t);return this._lastValue=o,this.richType.typeTransformer?this.richType.typeTransformer(o):o}const e=this.isConnected()?this._connectedPoint[0].getValue(t):this._getValueOrDefault(t);return this._lastValue=e,this.richType.typeTransformer?this.richType.typeTransformer(e):e}_getLastValue(){return this._lastValue}getClassName(){return"FlowGraphDataConnection"}serialize(t={}){super.serialize(t),t.richType={},this.richType.serialize(t.richType),t.optional=this._optional,tt("defaultValue",this._defaultValue,t)}}ht("FlowGraphDataConnection",Y);class $t{constructor(t){var e;this.config=t,this.uniqueId=D(),this.name=((e=this.config)==null?void 0:e.name)??this.getClassName(),this.dataInputs=[],this.dataOutputs=[]}_updateOutputs(t){}registerDataInput(t,e,o){const n=new Y(t,0,this,e,o);return this.dataInputs.push(n),n}registerDataOutput(t,e,o){const n=new Y(t,1,this,e,o);return this.dataOutputs.push(n),n}getDataInput(t){return this.dataInputs.find(e=>e.name===t)}getDataOutput(t){return this.dataOutputs.find(e=>e.name===t)}serialize(t={},e=tt){if(t.uniqueId=this.uniqueId,t.config={},this.config){const o=this.config,n=Object.keys(o);for(const s of n)e(s,o[s],t.config)}t.dataInputs=[],t.dataOutputs=[],t.className=this.getClassName();for(const o of this.dataInputs){const n={};o.serialize(n),t.dataInputs.push(n)}for(const o of this.dataOutputs){const n={};o.serialize(n),t.dataOutputs.push(n)}}deserialize(t){}_log(t,e,o){var n;(n=t.logger)==null||n.addLogItem({action:e,payload:o,className:this.getClassName(),uniqueId:this.uniqueId})}getClassName(){return"FlowGraphBlock"}}class z extends wt{constructor(){super(...arguments),this.priority=0}_isSingularConnection(){return!1}connectTo(t){super.connectTo(t),this._connectedPoint.sort((e,o)=>o.priority-e.priority)}_activateSignal(t){var e;if((e=t.logger)==null||e.addLogItem({action:"ActivateSignal",className:this._ownerBlock.getClassName(),uniqueId:this._ownerBlock.uniqueId,payload:{connectionType:this.connectionType,name:this.name}}),this.connectionType===0)t._notifyExecuteNode(this._ownerBlock),this._ownerBlock._execute(t,this),t._increaseExecutionId();else for(const o of this._connectedPoint)o._activateSignal(t)}}ht("FlowGraphSignalConnection",z);class J extends $t{constructor(t){super(t),this.priority=0,this.signalInputs=[],this.signalOutputs=[],this.in=this._registerSignalInput("in"),this.error=this._registerSignalOutput("error")}_registerSignalInput(t){const e=new z(t,0,this);return this.signalInputs.push(e),e}_registerSignalOutput(t){const e=new z(t,1,this);return this.signalOutputs.push(e),e}_unregisterSignalInput(t){const e=this.signalInputs.findIndex(o=>o.name===t);e!==-1&&(this.signalInputs[e].dispose(),this.signalInputs.splice(e,1))}_unregisterSignalOutput(t){const e=this.signalOutputs.findIndex(o=>o.name===t);e!==-1&&(this.signalOutputs[e].dispose(),this.signalOutputs.splice(e,1))}_reportError(t,e){this.error.payload=typeof e=="string"?new Error(e):e,this.error._activateSignal(t)}getSignalInput(t){return this.signalInputs.find(e=>e.name===t)}getSignalOutput(t){return this.signalOutputs.find(e=>e.name===t)}serialize(t={}){super.serialize(t),t.signalInputs=[],t.signalOutputs=[];for(const e of this.signalInputs){const o={};e.serialize(o),t.signalInputs.push(o)}for(const e of this.signalOutputs){const o={};e.serialize(o),t.signalOutputs.push(o)}}deserialize(t){for(let e=0;e<t.signalInputs.length;e++){const o=this.getSignalInput(t.signalInputs[e].name);if(o)o.deserialize(t.signalInputs[e]);else throw new Error("Could not find signal input with name "+t.signalInputs[e].name+" in block "+t.className)}for(let e=0;e<t.signalOutputs.length;e++){const o=this.getSignalOutput(t.signalOutputs[e].name);if(o)o.deserialize(t.signalOutputs[e]);else throw new Error("Could not find signal output with name "+t.signalOutputs[e].name+" in block "+t.className)}}getClassName(){return"FlowGraphExecutionBlock"}}class Kt{constructor(t){this.onEventTriggeredObservable=new U,this.sceneReadyTriggered=!1,this._pointerUnderMeshState={},this._startingTime=0,this._scene=t,this._initialize()}_initialize(){this._sceneReadyObserver=this._scene.onReadyObservable.add(()=>{this.sceneReadyTriggered||(this.onEventTriggeredObservable.notifyObservers({type:"SceneReady"}),this.sceneReadyTriggered=!0)}),this._sceneDisposeObserver=this._scene.onDisposeObservable.add(()=>{this.onEventTriggeredObservable.notifyObservers({type:"SceneDispose"})}),this._sceneOnBeforeRenderObserver=this._scene.onBeforeRenderObservable.add(()=>{const t=this._scene.getEngine().getDeltaTime()/1e3;this.onEventTriggeredObservable.notifyObservers({type:"SceneBeforeRender",payload:{timeSinceStart:this._startingTime,deltaTime:t}}),this._startingTime+=t}),this._meshPickedObserver=this._scene.onPointerObservable.add(t=>{this.onEventTriggeredObservable.notifyObservers({type:"MeshPick",payload:t})},rt.POINTERPICK),this._meshUnderPointerObserver=this._scene.onMeshUnderPointerUpdatedObservable.add(t=>{const e=t.pointerId,o=t.mesh,n=this._pointerUnderMeshState[e];!n&&o?this.onEventTriggeredObservable.notifyObservers({type:"PointerOver",payload:{pointerId:e,mesh:o}}):n&&!o?this.onEventTriggeredObservable.notifyObservers({type:"PointerOut",payload:{pointerId:e,mesh:n}}):n&&o&&n!==o&&(this.onEventTriggeredObservable.notifyObservers({type:"PointerOut",payload:{pointerId:e,mesh:n,over:o}}),this.onEventTriggeredObservable.notifyObservers({type:"PointerOver",payload:{pointerId:e,mesh:o,out:n}})),this._pointerUnderMeshState[e]=o},rt.POINTERMOVE)}dispose(){var t,e,o,n,s;(t=this._sceneDisposeObserver)==null||t.remove(),(e=this._sceneReadyObserver)==null||e.remove(),(o=this._sceneOnBeforeRenderObserver)==null||o.remove(),(n=this._meshPickedObserver)==null||n.remove(),(s=this._meshUnderPointerObserver)==null||s.remove(),this.onEventTriggeredObservable.clear()}}function dt(a,t){return!!(a.parent&&(a.parent===t||dt(a.parent,t)))}function de(a){if(a.getClassName)return a.getClassName()}function fe(a,t){return a===t&&(a==="Vector2"||a==="Vector3"||a==="Vector4"||a==="Quaternion")}function ye(a,t){return a===t&&(a==="Matrix"||a==="Matrix2D"||a==="Matrix3D")}function ge(a,t){return a==="FlowGraphInteger"&&t==="FlowGraphInteger"}function me(a,t){const e=typeof a=="number"||typeof(a==null?void 0:a.value)=="number";return e&&!t?!isNaN(Wt(a)):e}function Wt(a){return typeof a=="number"?a:a.value}var lt;(function(a){a[a.Stopped=0]="Stopped",a[a.Started=1]="Started"})(lt||(lt={}));class Ut{get state(){return this._state}set state(t){this._state=t,this.onStateChangedObservable.notifyObservers(t)}constructor(t){this.onStateChangedObservable=new U,this._eventBlocks={SceneReady:[],SceneDispose:[],SceneBeforeRender:[],MeshPick:[],PointerDown:[],PointerUp:[],PointerMove:[],PointerOver:[],PointerOut:[],SceneAfterRender:[],NoTrigger:[]},this._executionContexts=[],this._state=0,this._scene=t.scene,this._sceneEventCoordinator=new Kt(this._scene),this._coordinator=t.coordinator,this._eventObserver=this._sceneEventCoordinator.onEventTriggeredObservable.add(e=>{for(const o of this._executionContexts){const n=this._getContextualOrder(e.type,o);for(const s of n)if(!s._executeEvent(o,e.payload))break}switch(e.type){case"SceneReady":this._sceneEventCoordinator.sceneReadyTriggered=!0;break;case"SceneBeforeRender":for(const o of this._executionContexts)o._notifyOnTick(e.payload);break;case"SceneDispose":this.dispose();break}})}createContext(){const t=new _t({scene:this._scene,coordinator:this._coordinator});return this._executionContexts.push(t),t}getContext(t){return this._executionContexts[t]}addEventBlock(t){if((t.type==="PointerOver"||t.type==="PointerOut")&&(this._scene.constantlyUpdateMeshUnderPointer=!0),t.type!=="NoTrigger"&&this._eventBlocks[t.type].push(t),this.state===1)for(const e of this._executionContexts)t._startPendingTasks(e);else this.onStateChangedObservable.addOnce(e=>{if(e===1)for(const o of this._executionContexts)t._startPendingTasks(o)})}start(){this.state!==1&&(this._executionContexts.length===0&&this.createContext(),this.onStateChangedObservable.add(t=>{t===1&&(this._startPendingEvents(),this._scene.isReady(!0)&&this._sceneEventCoordinator.onEventTriggeredObservable.notifyObservers({type:"SceneReady"}))}),this.state=1)}_startPendingEvents(){for(const t of this._executionContexts)for(const e in this._eventBlocks){const o=this._getContextualOrder(e,t);for(const n of o)n._startPendingTasks(t)}}_getContextualOrder(t,e){const o=this._eventBlocks[t].sort((n,s)=>s.initPriority-n.initPriority);if(t==="MeshPick"){const n=[];for(const s of o){const i=s.asset.getValue(e);let c=0;for(;c<o.length;c++){const u=o[c].asset.getValue(e);if(i&&u&&dt(i,u))break}n.splice(c,0,s)}return n}return o}dispose(){var t;if(this.state!==0){this.state=0;for(const e of this._executionContexts)e._clearPendingBlocks();this._executionContexts.length=0;for(const e in this._eventBlocks)this._eventBlocks[e].length=0;(t=this._eventObserver)==null||t.remove(),this._sceneEventCoordinator.dispose()}}visitAllBlocks(t){const e=[],o=new Set;for(const n in this._eventBlocks)for(const s of this._eventBlocks[n])e.push(s),o.add(s.uniqueId);for(;e.length>0;){const n=e.pop();t(n);for(const s of n.dataInputs)for(const i of s._connectedPoint)o.has(i._ownerBlock.uniqueId)||(e.push(i._ownerBlock),o.add(i._ownerBlock.uniqueId));if(n instanceof J)for(const s of n.signalOutputs)for(const i of s._connectedPoint)o.has(i._ownerBlock.uniqueId)||(e.push(i._ownerBlock),o.add(i._ownerBlock.uniqueId))}}serialize(t={},e){t.allBlocks=[],this.visitAllBlocks(o=>{const n={};o.serialize(n),t.allBlocks.push(n)}),t.executionContexts=[];for(const o of this._executionContexts){const n={};o.serialize(n,e),t.executionContexts.push(n)}}}class E{constructor(t){this.config=t,this.dispatchEventsSynchronously=!0,this._flowGraphs=[],this._customEventsMap=new Map,this._eventExecutionCounter=new Map,this._executeOnNextFrame=[],this._eventUniqueId=0,this._disposeObserver=this.config.scene.onDisposeObservable.add(()=>{this.dispose()}),this._onBeforeRenderObserver=this.config.scene.onBeforeRenderObservable.add(()=>{this._eventExecutionCounter.clear();const o=this._executeOnNextFrame.slice(0);if(o.length)for(const n of o){this.notifyCustomEvent(n.id,n.data,!1);const s=this._executeOnNextFrame.findIndex(i=>i.uniqueId===n.uniqueId);s!==-1&&this._executeOnNextFrame.splice(s,1)}});let e=E.SceneCoordinators.get(this.config.scene);e||(e=[],E.SceneCoordinators.set(this.config.scene,e)),e.push(this)}createGraph(){const t=new Ut({scene:this.config.scene,coordinator:this});return this._flowGraphs.push(t),t}removeGraph(t){const e=this._flowGraphs.indexOf(t);e!==-1&&(t.dispose(),this._flowGraphs.splice(e,1))}start(){for(const t of this._flowGraphs)t.start()}dispose(){var o,n;for(const s of this._flowGraphs)s.dispose();this._flowGraphs.length=0,(o=this._disposeObserver)==null||o.remove(),(n=this._onBeforeRenderObserver)==null||n.remove();const t=E.SceneCoordinators.get(this.config.scene)??[],e=t.indexOf(this);e!==-1&&t.splice(e,1)}serialize(t,e){t._flowGraphs=[];for(const o of this._flowGraphs){const n={};o.serialize(n,e),t._flowGraphs.push(n)}t.dispatchEventsSynchronously=this.dispatchEventsSynchronously}get flowGraphs(){return this._flowGraphs}getCustomEventObservable(t){let e=this._customEventsMap.get(t);return e||(e=new U,this._customEventsMap.set(t,e)),e}notifyCustomEvent(t,e,o=!this.dispatchEventsSynchronously){if(o){this._executeOnNextFrame.push({id:t,data:e,uniqueId:this._eventUniqueId++});return}if(this._eventExecutionCounter.has(t)){const s=this._eventExecutionCounter.get(t);if(this._eventExecutionCounter.set(t,s+1),s>=E.MaxEventTypeExecutionPerFrame){s===E.MaxEventTypeExecutionPerFrame&&w.Warn(`FlowGraphCoordinator: Too many executions of event "${t}".`);return}}else this._eventExecutionCounter.set(t,1);const n=this._customEventsMap.get(t);n&&n.notifyObservers(e)}}E.MaxEventsPerType=30;E.MaxEventTypeExecutionPerFrame=30;E.SceneCoordinators=new Map;const j={};function Qt(a,t,e){j[`${a}/${t}`]=e}function Ht(a){switch(a){case"FlowGraphPlayAnimationBlock":return async()=>(await r(async()=>{const{FlowGraphPlayAnimationBlock:t}=await import("./flowGraphPlayAnimationBlock-B9FP0RfS.js");return{FlowGraphPlayAnimationBlock:t}},__vite__mapDeps([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]))).FlowGraphPlayAnimationBlock;case"FlowGraphStopAnimationBlock":return async()=>(await r(async()=>{const{FlowGraphStopAnimationBlock:t}=await import("./flowGraphStopAnimationBlock-ChXcK4cb.js");return{FlowGraphStopAnimationBlock:t}},__vite__mapDeps([32,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphStopAnimationBlock;case"FlowGraphPauseAnimationBlock":return async()=>(await r(async()=>{const{FlowGraphPauseAnimationBlock:t}=await import("./flowGraphPauseAnimationBlock-DAoY0eJc.js");return{FlowGraphPauseAnimationBlock:t}},__vite__mapDeps([33,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphPauseAnimationBlock;case"FlowGraphInterpolationBlock":return async()=>(await r(async()=>{const{FlowGraphInterpolationBlock:t}=await import("./flowGraphInterpolationBlock-CYcMKCGO.js");return{FlowGraphInterpolationBlock:t}},__vite__mapDeps([34,1,2,3,4,5,7,8,9,10,23,24,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,29,30,31]))).FlowGraphInterpolationBlock;case"FlowGraphSceneReadyEventBlock":return async()=>(await r(async()=>{const{FlowGraphSceneReadyEventBlock:t}=await import("./flowGraphSceneReadyEventBlock-Crn-rAIL.js");return{FlowGraphSceneReadyEventBlock:t}},__vite__mapDeps([35,4,23,24,3,2,9,5,1,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSceneReadyEventBlock;case"FlowGraphSceneTickEventBlock":return async()=>(await r(async()=>{const{FlowGraphSceneTickEventBlock:t}=await import("./flowGraphSceneTickEventBlock-gf3YDJRe.js");return{FlowGraphSceneTickEventBlock:t}},__vite__mapDeps([36,4,1,2,3,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSceneTickEventBlock;case"FlowGraphSendCustomEventBlock":return async()=>(await r(async()=>{const{FlowGraphSendCustomEventBlock:t}=await import("./flowGraphSendCustomEventBlock-Ba8XUM8G.js");return{FlowGraphSendCustomEventBlock:t}},__vite__mapDeps([37,4,23,24,3,2,9,5,1,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSendCustomEventBlock;case"FlowGraphReceiveCustomEventBlock":return async()=>(await r(async()=>{const{FlowGraphReceiveCustomEventBlock:t}=await import("./flowGraphReceiveCustomEventBlock-CId8KX4K.js");return{FlowGraphReceiveCustomEventBlock:t}},__vite__mapDeps([38,13,3,4,9,2,5,23,24,1,21,25,26,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphReceiveCustomEventBlock;case"FlowGraphMeshPickEventBlock":return async()=>(await r(async()=>{const{FlowGraphMeshPickEventBlock:t}=await import("./flowGraphMeshPickEventBlock-WsLHiXtY.js");return{FlowGraphMeshPickEventBlock:t}},__vite__mapDeps([39,21,3,4,1,2,5,23,24,9,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphMeshPickEventBlock;case"FlowGraphEBlock":return async()=>(await r(async()=>{const{FlowGraphEBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphEBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphEBlock;case"FlowGraphPIBlock":return async()=>(await r(async()=>{const{FlowGraphPiBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphPiBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphPiBlock;case"FlowGraphInfBlock":return async()=>(await r(async()=>{const{FlowGraphInfBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphInfBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphInfBlock;case"FlowGraphNaNBlock":return async()=>(await r(async()=>{const{FlowGraphNaNBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphNaNBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphNaNBlock;case"FlowGraphRandomBlock":return async()=>(await r(async()=>{const{FlowGraphRandomBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphRandomBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphRandomBlock;case"FlowGraphAddBlock":return async()=>(await r(async()=>{const{FlowGraphAddBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphAddBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAddBlock;case"FlowGraphSubtractBlock":return async()=>(await r(async()=>{const{FlowGraphSubtractBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphSubtractBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSubtractBlock;case"FlowGraphMultiplyBlock":return async()=>(await r(async()=>{const{FlowGraphMultiplyBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphMultiplyBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphMultiplyBlock;case"FlowGraphDivideBlock":return async()=>(await r(async()=>{const{FlowGraphDivideBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphDivideBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphDivideBlock;case"FlowGraphAbsBlock":return async()=>(await r(async()=>{const{FlowGraphAbsBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphAbsBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAbsBlock;case"FlowGraphSignBlock":return async()=>(await r(async()=>{const{FlowGraphSignBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphSignBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSignBlock;case"FlowGraphTruncBlock":return async()=>(await r(async()=>{const{FlowGraphTruncBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphTruncBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphTruncBlock;case"FlowGraphFloorBlock":return async()=>(await r(async()=>{const{FlowGraphFloorBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphFloorBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphFloorBlock;case"FlowGraphCeilBlock":return async()=>(await r(async()=>{const{FlowGraphCeilBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphCeilBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCeilBlock;case"FlowGraphRoundBlock":return async()=>(await r(async()=>{const{FlowGraphRoundBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphRoundBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphRoundBlock;case"FlowGraphFractBlock":return async()=>(await r(async()=>{const{FlowGraphFractionBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphFractionBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphFractionBlock;case"FlowGraphNegationBlock":return async()=>(await r(async()=>{const{FlowGraphNegationBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphNegationBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphNegationBlock;case"FlowGraphModuloBlock":return async()=>(await r(async()=>{const{FlowGraphModuloBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphModuloBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphModuloBlock;case"FlowGraphMinBlock":return async()=>(await r(async()=>{const{FlowGraphMinBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphMinBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphMinBlock;case"FlowGraphMaxBlock":return async()=>(await r(async()=>{const{FlowGraphMaxBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphMaxBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphMaxBlock;case"FlowGraphClampBlock":return async()=>(await r(async()=>{const{FlowGraphClampBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphClampBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphClampBlock;case"FlowGraphSaturateBlock":return async()=>(await r(async()=>{const{FlowGraphSaturateBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphSaturateBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSaturateBlock;case"FlowGraphMathInterpolationBlock":return async()=>(await r(async()=>{const{FlowGraphMathInterpolationBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphMathInterpolationBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphMathInterpolationBlock;case"FlowGraphEqualityBlock":return async()=>(await r(async()=>{const{FlowGraphEqualityBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphEqualityBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphEqualityBlock;case"FlowGraphLessThanBlock":return async()=>(await r(async()=>{const{FlowGraphLessThanBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphLessThanBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphLessThanBlock;case"FlowGraphLessThanOrEqualBlock":return async()=>(await r(async()=>{const{FlowGraphLessThanOrEqualBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphLessThanOrEqualBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphLessThanOrEqualBlock;case"FlowGraphGreaterThanBlock":return async()=>(await r(async()=>{const{FlowGraphGreaterThanBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphGreaterThanBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphGreaterThanBlock;case"FlowGraphGreaterThanOrEqualBlock":return async()=>(await r(async()=>{const{FlowGraphGreaterThanOrEqualBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphGreaterThanOrEqualBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphGreaterThanOrEqualBlock;case"FlowGraphIsNaNBlock":return async()=>(await r(async()=>{const{FlowGraphIsNanBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphIsNanBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphIsNanBlock;case"FlowGraphIsInfBlock":return async()=>(await r(async()=>{const{FlowGraphIsInfinityBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphIsInfinityBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphIsInfinityBlock;case"FlowGraphDegToRadBlock":return async()=>(await r(async()=>{const{FlowGraphDegToRadBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphDegToRadBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphDegToRadBlock;case"FlowGraphRadToDegBlock":return async()=>(await r(async()=>{const{FlowGraphRadToDegBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphRadToDegBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphRadToDegBlock;case"FlowGraphSinBlock":return async()=>(await r(async()=>{const{FlowGraphSinBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphSinBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSinBlock;case"FlowGraphCosBlock":return async()=>(await r(async()=>{const{FlowGraphCosBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphCosBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCosBlock;case"FlowGraphTanBlock":return async()=>(await r(async()=>{const{FlowGraphTanBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphTanBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphTanBlock;case"FlowGraphASinBlock":return async()=>(await r(async()=>{const{FlowGraphAsinBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphAsinBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAsinBlock;case"FlowGraphACosBlock":return async()=>(await r(async()=>{const{FlowGraphAcosBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphAcosBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAcosBlock;case"FlowGraphATanBlock":return async()=>(await r(async()=>{const{FlowGraphAtanBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphAtanBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAtanBlock;case"FlowGraphATan2Block":return async()=>(await r(async()=>{const{FlowGraphAtan2Block:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphAtan2Block:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAtan2Block;case"FlowGraphSinhBlock":return async()=>(await r(async()=>{const{FlowGraphSinhBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphSinhBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSinhBlock;case"FlowGraphCoshBlock":return async()=>(await r(async()=>{const{FlowGraphCoshBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphCoshBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCoshBlock;case"FlowGraphTanhBlock":return async()=>(await r(async()=>{const{FlowGraphTanhBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphTanhBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphTanhBlock;case"FlowGraphASinhBlock":return async()=>(await r(async()=>{const{FlowGraphAsinhBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphAsinhBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAsinhBlock;case"FlowGraphACoshBlock":return async()=>(await r(async()=>{const{FlowGraphAcoshBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphAcoshBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAcoshBlock;case"FlowGraphATanhBlock":return async()=>(await r(async()=>{const{FlowGraphAtanhBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphAtanhBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAtanhBlock;case"FlowGraphExponentialBlock":return async()=>(await r(async()=>{const{FlowGraphExpBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphExpBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphExpBlock;case"FlowGraphLogBlock":return async()=>(await r(async()=>{const{FlowGraphLogBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphLogBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphLogBlock;case"FlowGraphLog2Block":return async()=>(await r(async()=>{const{FlowGraphLog2Block:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphLog2Block:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphLog2Block;case"FlowGraphLog10Block":return async()=>(await r(async()=>{const{FlowGraphLog10Block:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphLog10Block:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphLog10Block;case"FlowGraphSquareRootBlock":return async()=>(await r(async()=>{const{FlowGraphSquareRootBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphSquareRootBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSquareRootBlock;case"FlowGraphPowerBlock":return async()=>(await r(async()=>{const{FlowGraphPowerBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphPowerBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphPowerBlock;case"FlowGraphCubeRootBlock":return async()=>(await r(async()=>{const{FlowGraphCubeRootBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphCubeRootBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCubeRootBlock;case"FlowGraphBitwiseAndBlock":return async()=>(await r(async()=>{const{FlowGraphBitwiseAndBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphBitwiseAndBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphBitwiseAndBlock;case"FlowGraphBitwiseOrBlock":return async()=>(await r(async()=>{const{FlowGraphBitwiseOrBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphBitwiseOrBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphBitwiseOrBlock;case"FlowGraphBitwiseNotBlock":return async()=>(await r(async()=>{const{FlowGraphBitwiseNotBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphBitwiseNotBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphBitwiseNotBlock;case"FlowGraphBitwiseXorBlock":return async()=>(await r(async()=>{const{FlowGraphBitwiseXorBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphBitwiseXorBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphBitwiseXorBlock;case"FlowGraphBitwiseLeftShiftBlock":return async()=>(await r(async()=>{const{FlowGraphBitwiseLeftShiftBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphBitwiseLeftShiftBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphBitwiseLeftShiftBlock;case"FlowGraphBitwiseRightShiftBlock":return async()=>(await r(async()=>{const{FlowGraphBitwiseRightShiftBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphBitwiseRightShiftBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphBitwiseRightShiftBlock;case"FlowGraphLengthBlock":return async()=>(await r(async()=>{const{FlowGraphLengthBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphLengthBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphLengthBlock;case"FlowGraphNormalizeBlock":return async()=>(await r(async()=>{const{FlowGraphNormalizeBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphNormalizeBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphNormalizeBlock;case"FlowGraphDotBlock":return async()=>(await r(async()=>{const{FlowGraphDotBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphDotBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphDotBlock;case"FlowGraphCrossBlock":return async()=>(await r(async()=>{const{FlowGraphCrossBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphCrossBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCrossBlock;case"FlowGraphRotate2DBlock":return async()=>(await r(async()=>{const{FlowGraphRotate2DBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphRotate2DBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphRotate2DBlock;case"FlowGraphRotate3DBlock":return async()=>(await r(async()=>{const{FlowGraphRotate3DBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphRotate3DBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphRotate3DBlock;case"FlowGraphTransposeBlock":return async()=>(await r(async()=>{const{FlowGraphTransposeBlock:t}=await import("./flowGraphMatrixMathBlocks-Dg7VLPFP.js");return{FlowGraphTransposeBlock:t}},__vite__mapDeps([45,1,2,3,4,5,43,42,41,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphTransposeBlock;case"FlowGraphDeterminantBlock":return async()=>(await r(async()=>{const{FlowGraphDeterminantBlock:t}=await import("./flowGraphMatrixMathBlocks-Dg7VLPFP.js");return{FlowGraphDeterminantBlock:t}},__vite__mapDeps([45,1,2,3,4,5,43,42,41,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphDeterminantBlock;case"FlowGraphInvertMatrixBlock":return async()=>(await r(async()=>{const{FlowGraphInvertMatrixBlock:t}=await import("./flowGraphMatrixMathBlocks-Dg7VLPFP.js");return{FlowGraphInvertMatrixBlock:t}},__vite__mapDeps([45,1,2,3,4,5,43,42,41,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphInvertMatrixBlock;case"FlowGraphMatrixMultiplicationBlock":return async()=>(await r(async()=>{const{FlowGraphMatrixMultiplicationBlock:t}=await import("./flowGraphMatrixMathBlocks-Dg7VLPFP.js");return{FlowGraphMatrixMultiplicationBlock:t}},__vite__mapDeps([45,1,2,3,4,5,43,42,41,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphMatrixMultiplicationBlock;case"FlowGraphBranchBlock":return async()=>(await r(async()=>{const{FlowGraphBranchBlock:t}=await import("./flowGraphBranchBlock-CPQ1Lj8X.js");return{FlowGraphBranchBlock:t}},__vite__mapDeps([46,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphBranchBlock;case"FlowGraphSetDelayBlock":return async()=>(await r(async()=>{const{FlowGraphSetDelayBlock:t}=await import("./flowGraphSetDelayBlock-BA13Fleb.js");return{FlowGraphSetDelayBlock:t}},__vite__mapDeps([47,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSetDelayBlock;case"FlowGraphCancelDelayBlock":return async()=>(await r(async()=>{const{FlowGraphCancelDelayBlock:t}=await import("./flowGraphCancelDelayBlock-CiPuCgGz.js");return{FlowGraphCancelDelayBlock:t}},__vite__mapDeps([48,4,1,2,3,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCancelDelayBlock;case"FlowGraphCallCounterBlock":return async()=>(await r(async()=>{const{FlowGraphCallCounterBlock:t}=await import("./flowGraphCounterBlock-OjaFZjW5.js");return{FlowGraphCallCounterBlock:t}},__vite__mapDeps([49,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCallCounterBlock;case"FlowGraphDebounceBlock":return async()=>(await r(async()=>{const{FlowGraphDebounceBlock:t}=await import("./flowGraphDebounceBlock-fxe8WoxT.js");return{FlowGraphDebounceBlock:t}},__vite__mapDeps([50,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphDebounceBlock;case"FlowGraphThrottleBlock":return async()=>(await r(async()=>{const{FlowGraphThrottleBlock:t}=await import("./flowGraphThrottleBlock-B-yO7rgE.js");return{FlowGraphThrottleBlock:t}},__vite__mapDeps([51,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphThrottleBlock;case"FlowGraphDoNBlock":return async()=>(await r(async()=>{const{FlowGraphDoNBlock:t}=await import("./flowGraphDoNBlock-C4vVyovK.js");return{FlowGraphDoNBlock:t}},__vite__mapDeps([52,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphDoNBlock;case"FlowGraphFlipFlopBlock":return async()=>(await r(async()=>{const{FlowGraphFlipFlopBlock:t}=await import("./flowGraphFlipFlopBlock-BSjrwcqq.js");return{FlowGraphFlipFlopBlock:t}},__vite__mapDeps([53,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphFlipFlopBlock;case"FlowGraphForLoopBlock":return async()=>(await r(async()=>{const{FlowGraphForLoopBlock:t}=await import("./flowGraphForLoopBlock-jI9z_K5X.js");return{FlowGraphForLoopBlock:t}},__vite__mapDeps([54,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphForLoopBlock;case"FlowGraphMultiGateBlock":return async()=>(await r(async()=>{const{FlowGraphMultiGateBlock:t}=await import("./flowGraphMultiGateBlock-Cy-tdSZ3.js");return{FlowGraphMultiGateBlock:t}},__vite__mapDeps([55,4,1,2,3,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphMultiGateBlock;case"FlowGraphSequenceBlock":return async()=>(await r(async()=>{const{FlowGraphSequenceBlock:t}=await import("./flowGraphSequenceBlock-Bcye5mDB.js");return{FlowGraphSequenceBlock:t}},__vite__mapDeps([56,4,23,24,3,2,9,5,1,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSequenceBlock;case"FlowGraphSwitchBlock":return async()=>(await r(async()=>{const{FlowGraphSwitchBlock:t}=await import("./flowGraphSwitchBlock-CfuObCnO.js");return{FlowGraphSwitchBlock:t}},__vite__mapDeps([57,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSwitchBlock;case"FlowGraphWaitAllBlock":return async()=>(await r(async()=>{const{FlowGraphWaitAllBlock:t}=await import("./flowGraphWaitAllBlock-VXxeVnO_.js");return{FlowGraphWaitAllBlock:t}},__vite__mapDeps([58,4,1,2,3,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphWaitAllBlock;case"FlowGraphWhileLoopBlock":return async()=>(await r(async()=>{const{FlowGraphWhileLoopBlock:t}=await import("./flowGraphWhileLoopBlock-BYmEfSWz.js");return{FlowGraphWhileLoopBlock:t}},__vite__mapDeps([59,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphWhileLoopBlock;case"FlowGraphConsoleLogBlock":return async()=>(await r(async()=>{const{FlowGraphConsoleLogBlock:t}=await import("./flowGraphConsoleLogBlock-Cz7IRQyV.js");return{FlowGraphConsoleLogBlock:t}},__vite__mapDeps([60,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphConsoleLogBlock;case"FlowGraphConditionalBlock":return async()=>(await r(async()=>{const{FlowGraphConditionalDataBlock:t}=await import("./flowGraphConditionalDataBlock-Btzamqcx.js");return{FlowGraphConditionalDataBlock:t}},__vite__mapDeps([61,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphConditionalDataBlock;case"FlowGraphConstantBlock":return async()=>(await r(async()=>{const{FlowGraphConstantBlock:t}=await import("./flowGraphConstantBlock-BV1ij11S.js");return{FlowGraphConstantBlock:t}},__vite__mapDeps([62,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphConstantBlock;case"FlowGraphTransformCoordinatesSystemBlock":return async()=>(await r(async()=>{const{FlowGraphTransformCoordinatesSystemBlock:t}=await import("./flowGraphTransformCoordinatesSystemBlock-D3zEJ4A-.js");return{FlowGraphTransformCoordinatesSystemBlock:t}},__vite__mapDeps([63,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphTransformCoordinatesSystemBlock;case"FlowGraphGetAssetBlock":return async()=>(await r(async()=>{const{FlowGraphGetAssetBlock:t}=await import("./flowGraphGetAssetBlock-BeMIiO3t.js");return{FlowGraphGetAssetBlock:t}},__vite__mapDeps([64,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphGetAssetBlock;case"FlowGraphGetPropertyBlock":return async()=>(await r(async()=>{const{FlowGraphGetPropertyBlock:t}=await import("./flowGraphGetPropertyBlock-CAOmAIk8.js");return{FlowGraphGetPropertyBlock:t}},__vite__mapDeps([65,1,2,3,4,5,42,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphGetPropertyBlock;case"FlowGraphSetPropertyBlock":return async()=>(await r(async()=>{const{FlowGraphSetPropertyBlock:t}=await import("./flowGraphSetPropertyBlock-DxJBj-a9.js");return{FlowGraphSetPropertyBlock:t}},__vite__mapDeps([66,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSetPropertyBlock;case"FlowGraphGetVariableBlock":return async()=>(await r(async()=>{const{FlowGraphGetVariableBlock:t}=await import("./flowGraphGetVariableBlock-BaRkHNJ_.js");return{FlowGraphGetVariableBlock:t}},__vite__mapDeps([67,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphGetVariableBlock;case"FlowGraphSetVariableBlock":return async()=>(await r(async()=>{const{FlowGraphSetVariableBlock:t}=await import("./flowGraphSetVariableBlock-BG0Kfa6g.js");return{FlowGraphSetVariableBlock:t}},__vite__mapDeps([68,4,1,2,3,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphSetVariableBlock;case"FlowGraphJsonPointerParserBlock":return async()=>(await r(async()=>{const{FlowGraphJsonPointerParserBlock:t}=await import("./flowGraphJsonPointerParserBlock-CZ93bSRG.js");return{FlowGraphJsonPointerParserBlock:t}},__vite__mapDeps([69,1,2,3,4,5,42,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphJsonPointerParserBlock;case"FlowGraphLeadingZerosBlock":return async()=>(await r(async()=>{const{FlowGraphLeadingZerosBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphLeadingZerosBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphLeadingZerosBlock;case"FlowGraphTrailingZerosBlock":return async()=>(await r(async()=>{const{FlowGraphTrailingZerosBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphTrailingZerosBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphTrailingZerosBlock;case"FlowGraphOneBitsCounterBlock":return async()=>(await r(async()=>{const{FlowGraphOneBitsCounterBlock:t}=await import("./flowGraphMathBlocks-HeZhspxf.js");return{FlowGraphOneBitsCounterBlock:t}},__vite__mapDeps([40,4,1,2,3,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphOneBitsCounterBlock;case"FlowGraphCombineVector2Block":return async()=>(await r(async()=>{const{FlowGraphCombineVector2Block:t}=await import("./flowGraphMathCombineExtractBlocks-Dg02KNQM.js");return{FlowGraphCombineVector2Block:t}},__vite__mapDeps([70,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCombineVector2Block;case"FlowGraphCombineVector3Block":return async()=>(await r(async()=>{const{FlowGraphCombineVector3Block:t}=await import("./flowGraphMathCombineExtractBlocks-Dg02KNQM.js");return{FlowGraphCombineVector3Block:t}},__vite__mapDeps([70,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCombineVector3Block;case"FlowGraphCombineVector4Block":return async()=>(await r(async()=>{const{FlowGraphCombineVector4Block:t}=await import("./flowGraphMathCombineExtractBlocks-Dg02KNQM.js");return{FlowGraphCombineVector4Block:t}},__vite__mapDeps([70,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCombineVector4Block;case"FlowGraphCombineMatrixBlock":return async()=>(await r(async()=>{const{FlowGraphCombineMatrixBlock:t}=await import("./flowGraphMathCombineExtractBlocks-Dg02KNQM.js");return{FlowGraphCombineMatrixBlock:t}},__vite__mapDeps([70,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCombineMatrixBlock;case"FlowGraphExtractVector2Block":return async()=>(await r(async()=>{const{FlowGraphExtractVector2Block:t}=await import("./flowGraphMathCombineExtractBlocks-Dg02KNQM.js");return{FlowGraphExtractVector2Block:t}},__vite__mapDeps([70,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphExtractVector2Block;case"FlowGraphExtractVector3Block":return async()=>(await r(async()=>{const{FlowGraphExtractVector3Block:t}=await import("./flowGraphMathCombineExtractBlocks-Dg02KNQM.js");return{FlowGraphExtractVector3Block:t}},__vite__mapDeps([70,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphExtractVector3Block;case"FlowGraphExtractVector4Block":return async()=>(await r(async()=>{const{FlowGraphExtractVector4Block:t}=await import("./flowGraphMathCombineExtractBlocks-Dg02KNQM.js");return{FlowGraphExtractVector4Block:t}},__vite__mapDeps([70,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphExtractVector4Block;case"FlowGraphExtractMatrixBlock":return async()=>(await r(async()=>{const{FlowGraphExtractMatrixBlock:t}=await import("./flowGraphMathCombineExtractBlocks-Dg02KNQM.js");return{FlowGraphExtractMatrixBlock:t}},__vite__mapDeps([70,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphExtractMatrixBlock;case"FlowGraphTransformVectorBlock":return async()=>(await r(async()=>{const{FlowGraphTransformBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphTransformBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphTransformBlock;case"FlowGraphTransformCoordinatesBlock":return async()=>(await r(async()=>{const{FlowGraphTransformCoordinatesBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphTransformCoordinatesBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphTransformCoordinatesBlock;case"FlowGraphConjugateBlock":return async()=>(await r(async()=>{const{FlowGraphConjugateBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphConjugateBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphConjugateBlock;case"FlowGraphAngleBetweenBlock":return async()=>(await r(async()=>{const{FlowGraphAngleBetweenBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphAngleBetweenBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAngleBetweenBlock;case"FlowGraphQuaternionFromAxisAngleBlock":return async()=>(await r(async()=>{const{FlowGraphQuaternionFromAxisAngleBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphQuaternionFromAxisAngleBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphQuaternionFromAxisAngleBlock;case"FlowGraphAxisAngleFromQuaternionBlock":return async()=>(await r(async()=>{const{FlowGraphAxisAngleFromQuaternionBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphAxisAngleFromQuaternionBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphAxisAngleFromQuaternionBlock;case"FlowGraphQuaternionFromDirectionsBlock":return async()=>(await r(async()=>{const{FlowGraphQuaternionFromDirectionsBlock:t}=await import("./flowGraphVectorMathBlocks-BGV5dKz9.js");return{FlowGraphQuaternionFromDirectionsBlock:t}},__vite__mapDeps([44,1,2,3,4,5,41,42,43,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphQuaternionFromDirectionsBlock;case"FlowGraphMatrixDecompose":return async()=>(await r(async()=>{const{FlowGraphMatrixDecomposeBlock:t}=await import("./flowGraphMatrixMathBlocks-Dg7VLPFP.js");return{FlowGraphMatrixDecomposeBlock:t}},__vite__mapDeps([45,1,2,3,4,5,43,42,41,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphMatrixDecomposeBlock;case"FlowGraphMatrixCompose":return async()=>(await r(async()=>{const{FlowGraphMatrixComposeBlock:t}=await import("./flowGraphMatrixMathBlocks-Dg7VLPFP.js");return{FlowGraphMatrixComposeBlock:t}},__vite__mapDeps([45,1,2,3,4,5,43,42,41,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphMatrixComposeBlock;case"FlowGraphBooleanToFloat":return async()=>(await r(async()=>{const{FlowGraphBooleanToFloat:t}=await import("./flowGraphTypeToTypeBlocks-DdLy9OXZ.js");return{FlowGraphBooleanToFloat:t}},__vite__mapDeps([71,43,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphBooleanToFloat;case"FlowGraphBooleanToInt":return async()=>(await r(async()=>{const{FlowGraphBooleanToInt:t}=await import("./flowGraphTypeToTypeBlocks-DdLy9OXZ.js");return{FlowGraphBooleanToInt:t}},__vite__mapDeps([71,43,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphBooleanToInt;case"FlowGraphFloatToBoolean":return async()=>(await r(async()=>{const{FlowGraphFloatToBoolean:t}=await import("./flowGraphTypeToTypeBlocks-DdLy9OXZ.js");return{FlowGraphFloatToBoolean:t}},__vite__mapDeps([71,43,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphFloatToBoolean;case"FlowGraphIntToBoolean":return async()=>(await r(async()=>{const{FlowGraphIntToBoolean:t}=await import("./flowGraphTypeToTypeBlocks-DdLy9OXZ.js");return{FlowGraphIntToBoolean:t}},__vite__mapDeps([71,43,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphIntToBoolean;case"FlowGraphIntToFloat":return async()=>(await r(async()=>{const{FlowGraphIntToFloat:t}=await import("./flowGraphTypeToTypeBlocks-DdLy9OXZ.js");return{FlowGraphIntToFloat:t}},__vite__mapDeps([71,43,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphIntToFloat;case"FlowGraphFloatToInt":return async()=>(await r(async()=>{const{FlowGraphFloatToInt:t}=await import("./flowGraphTypeToTypeBlocks-DdLy9OXZ.js");return{FlowGraphFloatToInt:t}},__vite__mapDeps([71,43,42,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphFloatToInt;case"FlowGraphEasingBlock":return async()=>(await r(async()=>{const{FlowGraphEasingBlock:t}=await import("./flowGraphEasingBlock-C_-3ZhLh.js");return{FlowGraphEasingBlock:t}},__vite__mapDeps([72,73,74,4,3,1,2,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphEasingBlock;case"FlowGraphBezierCurveEasing":return async()=>(await r(async()=>{const{FlowGraphBezierCurveEasingBlock:t}=await import("./flowGraphBezierCurveEasingBlock--1XIKCm2.js");return{FlowGraphBezierCurveEasingBlock:t}},__vite__mapDeps([75,73,74,4,3,1,2,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphBezierCurveEasingBlock;case"FlowGraphPointerOverEventBlock":return async()=>(await r(async()=>{const{FlowGraphPointerOverEventBlock:t}=await import("./flowGraphPointerOverEventBlock-wKWlgMPO.js");return{FlowGraphPointerOverEventBlock:t}},__vite__mapDeps([76,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphPointerOverEventBlock;case"FlowGraphPointerOutEventBlock":return async()=>(await r(async()=>{const{FlowGraphPointerOutEventBlock:t}=await import("./flowGraphPointerOutEventBlock-BpW1yJKn.js");return{FlowGraphPointerOutEventBlock:t}},__vite__mapDeps([77,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphPointerOutEventBlock;case"FlowGraphContextBlock":return async()=>(await r(async()=>{const{FlowGraphContextBlock:t}=await import("./flowGraphContextBlock-DeEwvFEY.js");return{FlowGraphContextBlock:t}},__vite__mapDeps([78,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphContextBlock;case"FlowGraphArrayIndexBlock":return async()=>(await r(async()=>{const{FlowGraphArrayIndexBlock:t}=await import("./flowGraphArrayIndexBlock-3XRt4h8L.js");return{FlowGraphArrayIndexBlock:t}},__vite__mapDeps([79,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphArrayIndexBlock;case"FlowGraphCodeExecutionBlock":return async()=>(await r(async()=>{const{FlowGraphCodeExecutionBlock:t}=await import("./flowGraphCodeExecutionBlock-BpjYYwgf.js");return{FlowGraphCodeExecutionBlock:t}},__vite__mapDeps([80,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphCodeExecutionBlock;case"FlowGraphIndexOfBlock":return async()=>(await r(async()=>{const{FlowGraphIndexOfBlock:t}=await import("./flowGraphIndexOfBlock-D1XW4kc4.js");return{FlowGraphIndexOfBlock:t}},__vite__mapDeps([81,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphIndexOfBlock;case"FlowGraphFunctionReference":return async()=>(await r(async()=>{const{FlowGraphFunctionReferenceBlock:t}=await import("./flowGraphFunctionReferenceBlock-CLhl7P_Q.js");return{FlowGraphFunctionReferenceBlock:t}},__vite__mapDeps([82,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphFunctionReferenceBlock;case"FlowGraphDataSwitchBlock":return async()=>(await r(async()=>{const{FlowGraphDataSwitchBlock:t}=await import("./flowGraphDataSwitchBlock-BUqG5JEg.js");return{FlowGraphDataSwitchBlock:t}},__vite__mapDeps([83,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphDataSwitchBlock;default:if(j[a])return j[a];throw new Error(`Unknown block name ${a}`)}}class Jt extends J{constructor(t){super(t),this.out=this._registerSignalOutput("out")}}class Zt extends Jt{constructor(t,e){if(super(t),this._eventsSignalOutputs={},this.done=this._registerSignalOutput("done"),e)for(const o of e)this._eventsSignalOutputs[o]=this._registerSignalOutput(o+"Event")}_executeOnTick(t){}_startPendingTasks(t){t._getExecutionVariable(this,"_initialized",!1)&&(this._cancelPendingTasks(t),this._resetAfterCanceled(t)),this._preparePendingTasks(t),t._addPendingBlock(this),this.out._activateSignal(t),t._setExecutionVariable(this,"_initialized",!0)}_resetAfterCanceled(t){t._deleteExecutionVariable(this,"_initialized"),t._removePendingBlock(this)}}class Xt extends Zt{constructor(){super(...arguments),this.initPriority=0,this.type="NoTrigger"}_execute(t){t._notifyExecuteNode(this),this.done._activateSignal(t)}}function Yt(a,t){for(const e of a)for(const o of e.dataOutputs)if(o.uniqueId===t)return o;throw new Error("Could not find data out connection with unique id "+t)}function zt(a,t){for(const e of a)if(e instanceof J){for(const o of e.signalInputs)if(o.uniqueId===t)return o}throw new Error("Could not find signal in connection with unique id "+t)}async function jt(a,t){const e=await Promise.all(a.allBlocks.map(async o=>await Ht(o.className)()));return te(a,t,e)}function te(a,t,e){const o=t.coordinator.createGraph(),n=[],s=t.valueParseFunction??H;for(let i=0;i<a.allBlocks.length;i++){const c=a.allBlocks[i],l=oe(c,{scene:t.coordinator.config.scene,pathConverter:t.pathConverter,assetsContainer:t.coordinator.config.scene,valueParseFunction:s},e[i]);n.push(l),l instanceof Xt&&o.addEventBlock(l)}for(const i of n){for(const c of i.dataInputs)for(const l of c.connectedPointIds){const u=Yt(n,l);c.connectTo(u)}if(i instanceof J)for(const c of i.signalOutputs)for(const l of c.connectedPointIds){const u=zt(n,l);c.connectTo(u)}}for(const i of a.executionContexts)ee(i,{graph:o,valueParseFunction:s},a.rightHanded);return o}function ee(a,t,e){var i,c,l,u,m,f,A,R,L,S;const o=t.graph.createContext();a.enableLogging&&(o.enableLogging=!0),o.treatDataAsRightHanded=e||!1;const n=t.valueParseFunction??H;o.uniqueId=a.uniqueId;const s=o.getScene();if(a._assetsContext){const p=a._assetsContext,v={meshes:(i=p.meshes)==null?void 0:i.map(h=>s.getMeshById(h)),lights:(c=p.lights)==null?void 0:c.map(h=>s.getLightByName(h)),cameras:(l=p.cameras)==null?void 0:l.map(h=>s.getCameraByName(h)),materials:(u=p.materials)==null?void 0:u.map(h=>s.getMaterialById(h)),textures:(m=p.textures)==null?void 0:m.map(h=>s.getTextureByName(h)),animations:(f=p.animations)==null?void 0:f.map(h=>s.animations.find(Q=>Q.name===h)),skeletons:(A=p.skeletons)==null?void 0:A.map(h=>s.getSkeletonByName(h)),particleSystems:(R=p.particleSystems)==null?void 0:R.map(h=>s.getParticleSystemById(h)),animationGroups:(L=p.animationGroups)==null?void 0:L.map(h=>s.getAnimationGroupByName(h)),transformNodes:(S=p.transformNodes)==null?void 0:S.map(h=>s.getTransformNodeById(h)),rootNodes:[],multiMaterials:[],morphTargetManagers:[],geometries:[],actionManagers:[],environmentTexture:null,postProcesses:[],sounds:null,effectLayers:[],layers:[],reflectionProbes:[],lensFlareSystems:[],proceduralTextures:[],getNodes:function(){throw new Error("Function not implemented.")}};o.assetsContext=v}for(const p in a._userVariables){const v=n(p,a._userVariables,o.assetsContext,s);o.userVariables[p]=v}for(const p in a._connectionValues){const v=n(p,a._connectionValues,o.assetsContext,s);o._setConnectionValueByKey(p,v)}return o}function oe(a,t,e){const o={},n=t.valueParseFunction??H;if(a.config)for(const i in a.config)o[i]=n(i,a.config,t.assetsContainer||t.scene,t.scene);if(Nt(a.className)){if(!t.pathConverter)throw new Error("Path converter is required for this block");o.pathConverter=t.pathConverter}const s=new e(o);s.uniqueId=a.uniqueId;for(let i=0;i<a.dataInputs.length;i++){const c=s.getDataInput(a.dataInputs[i].name);if(c)c.deserialize(a.dataInputs[i]);else throw new Error("Could not find data input with name "+a.dataInputs[i].name+" in block "+a.className)}for(let i=0;i<a.dataOutputs.length;i++){const c=s.getDataOutput(a.dataOutputs[i].name);if(c)c.deserialize(a.dataOutputs[i]);else throw new Error("Could not find data output with name "+a.dataOutputs[i].name+" in block "+a.className)}return s.metadata=a.metadata,s.deserialize&&s.deserialize(a),s}const ne={float:{length:1,flowGraphType:"number",elementType:"number"},bool:{length:1,flowGraphType:"boolean",elementType:"boolean"},float2:{length:2,flowGraphType:"Vector2",elementType:"number"},float3:{length:3,flowGraphType:"Vector3",elementType:"number"},float4:{length:4,flowGraphType:"Vector4",elementType:"number"},float4x4:{length:16,flowGraphType:"Matrix",elementType:"number"},float2x2:{length:4,flowGraphType:"Matrix2D",elementType:"number"},float3x3:{length:9,flowGraphType:"Matrix3D",elementType:"number"},int:{length:1,flowGraphType:"FlowGraphInteger",elementType:"number"}};class ae{constructor(t,e,o=60){this._interactivityGraph=t,this._gltf=e,this._animationTargetFps=o,this._types=[],this._mappings=[],this._staticVariables=[],this._events=[],this._internalEventsCounter=0,this._nodes=[],this._parseTypes(),this._parseDeclarations(),this._parseVariables(),this._parseEvents(),this._parseNodes()}get arrays(){return{types:this._types,mappings:this._mappings,staticVariables:this._staticVariables,events:this._events,nodes:this._nodes}}_parseTypes(){if(this._interactivityGraph.types)for(const t of this._interactivityGraph.types)this._types.push(ne[t.signature])}_parseDeclarations(){if(this._interactivityGraph.declarations)for(const t of this._interactivityGraph.declarations){const e=bt(t);if(!e)throw w.Error(["No mapping found for declaration",t]),new Error("Error parsing declarations");this._mappings.push({flowGraphMapping:e,fullOperationName:t.extension?t.op+":"+t.extension:t.op})}}_parseVariables(){if(this._interactivityGraph.variables)for(const t of this._interactivityGraph.variables){const e=this._parseVariable(t);this._staticVariables.push(e)}}_parseVariable(t,e){const o=this._types[t.type];if(!o)throw w.Error(["No type found for variable",t]),new Error("Error parsing variables");if(t.value&&t.value.length!==o.length)throw w.Error(["Invalid value length for variable",t,o]),new Error("Error parsing variables");const n=t.value||[];if(!n.length)switch(o.flowGraphType){case"boolean":n.push(!1);break;case"FlowGraphInteger":n.push(0);break;case"number":n.push(NaN);break;case"Vector2":n.push(NaN,NaN);break;case"Vector3":n.push(NaN,NaN,NaN);break;case"Vector4":case"Matrix2D":case"Quaternion":n.fill(NaN,0,4);break;case"Matrix":n.fill(NaN,0,16);break;case"Matrix3D":n.fill(NaN,0,9);break}return o.elementType==="number"&&typeof n[0]=="string"&&(n[0]=parseFloat(n[0])),{type:o.flowGraphType,value:e?e(n,this):n}}_parseEvents(){if(this._interactivityGraph.events)for(const t of this._interactivityGraph.events){const e={eventId:t.id||"internalEvent_"+this._internalEventsCounter++};t.values&&(e.eventData=Object.keys(t.values).map(o=>{var c;const n=(c=t.values)==null?void 0:c[o];if(!n)throw w.Error(["No value found for event key",o]),new Error("Error parsing events");const s=this._types[n.type];if(!s)throw w.Error(["No type found for event value",n]),new Error("Error parsing events");const i=typeof n.value<"u"?this._parseVariable(n):void 0;return{id:o,type:s.flowGraphType,eventData:!0,value:i}})),this._events.push(e)}}_parseNodes(){var t;if(this._interactivityGraph.nodes)for(const e of this._interactivityGraph.nodes){if(typeof e.declaration!="number")throw w.Error(["No declaration found for node",e]),new Error("Error parsing nodes");const o=this._mappings[e.declaration];if(!o)throw w.Error(["No mapping found for node",e]),new Error("Error parsing nodes");if(o.flowGraphMapping.validation){const s=o.flowGraphMapping.validation(e,this._interactivityGraph,this._gltf);if(!s.valid)throw new Error(`Error validating interactivity node ${(t=this._interactivityGraph.declarations)==null?void 0:t[e.declaration].op} - ${s.error}`)}const n=[];for(const s of o.flowGraphMapping.blocks){const i=this._getEmptyBlock(s,o.fullOperationName);this._parseNodeConfiguration(e,i,o.flowGraphMapping,s),n.push(i)}this._nodes.push({blocks:n,fullOperationName:o.fullOperationName})}}_getEmptyBlock(t,e){return{uniqueId:D(),className:t,dataInputs:[],dataOutputs:[],signalInputs:[],signalOutputs:[],config:{},type:e,metadata:{}}}_parseNodeConfiguration(t,e,o,n){var i;const s=t.configuration;if(s)for(const c in s){const l=s[c];if(!l)throw new Error("Error parsing node configuration");const u=(i=o.configuration)==null?void 0:i[c];if(u&&u.toBlock?u.toBlock===n:o.blocks.indexOf(n)===0){let f=u==null?void 0:u.defaultValue;l!=null&&l.value&&(f=l.value),u!=null&&u.isArray||(f.length!==1&&w.Warn(`Invalid non-array value length: ${f.length}`),f=f[0]),u!=null&&u.dataTransformer&&(f=u.dataTransformer(f,this)),f!==void 0&&(e.config[(u==null?void 0:u.name)||c]={value:f})}}}_parseNodeConnections(t){var e,o,n,s,i,c,l,u,m,f,A,R,L,S,p,v,h,Q,et;for(let N=0;N<this._nodes.length;N++){const I=(e=this._interactivityGraph.nodes)==null?void 0:e[N];if(!I)throw w.Error(["No node found for interactivity node",this._nodes[N]]),new Error("Error parsing node connections");const T=this._nodes[N],F=this._mappings[I.declaration];if(!F)throw w.Error(["No mapping found for node",I]),new Error("Error parsing node connections");const ot=I.flows||{},gt=Object.keys(ot).sort();for(const d of gt){const y=ot[d],_=(n=(o=F.flowGraphMapping.outputs)==null?void 0:o.flows)==null?void 0:n[d],x=(_==null?void 0:_.name)||d,M=this._createNewSocketConnection(x,!0);(_&&_.toBlock&&T.blocks.find(B=>B.className===_.toBlock)||T.blocks[0]).signalOutputs.push(M);const Z=y.node,g=this._nodes[Z];if(!g)throw w.Error(["No node found for input node id",Z]),new Error("Error parsing node connections");const V=at(g.fullOperationName);if(!V)throw w.Error(["No mapping found for input node",g]),new Error("Error parsing node connections");let k=(i=(s=V.inputs)==null?void 0:s.flows)==null?void 0:i[y.socket||"in"],P=!1;if(!k)for(const B in(c=V.inputs)==null?void 0:c.flows)B.startsWith("[")&&B.endsWith("]")&&(P=!0,k=(u=(l=V.inputs)==null?void 0:l.flows)==null?void 0:u[B]);const G=k?P?k.name.replace("$1",y.socket||""):k.name:y.socket||"in",$=k&&k.toBlock&&g.blocks.find(B=>B.className===k.toBlock)||g.blocks[0];let C=$.signalInputs.find(B=>B.name===G);C||(C=this._createNewSocketConnection(G),$.signalInputs.push(C)),C.connectedPointIds.push(M.uniqueId),M.connectedPointIds.push(C.uniqueId)}const nt=I.values||{},mt=Object.keys(nt);for(const d of mt){const y=nt[d];let _=(f=(m=F.flowGraphMapping.inputs)==null?void 0:m.values)==null?void 0:f[d],x=!1;if(!_)for(const g in(A=F.flowGraphMapping.inputs)==null?void 0:A.values)g.startsWith("[")&&g.endsWith("]")&&(x=!0,_=(L=(R=F.flowGraphMapping.inputs)==null?void 0:R.values)==null?void 0:L[g]);const M=_?x?_.name.replace("$1",d):_.name:d,q=this._createNewSocketConnection(M);if((_&&_.toBlock&&T.blocks.find(g=>g.className===_.toBlock)||T.blocks[0]).dataInputs.push(q),y.value!==void 0){const g=this._parseVariable(y,_&&_.dataTransformer);t._connectionValues[q.uniqueId]=g}else if(typeof y.node<"u"){const g=y.node,V=y.socket||"value",k=this._nodes[g];if(!k)throw w.Error(["No node found for output socket reference",y]),new Error("Error parsing node connections");const P=at(k.fullOperationName);if(!P)throw w.Error(["No mapping found for output socket reference",y]),new Error("Error parsing node connections");let G=(p=(S=P.outputs)==null?void 0:S.values)==null?void 0:p[V],$=!1;if(!G)for(const b in(v=P.outputs)==null?void 0:v.values)b.startsWith("[")&&b.endsWith("]")&&($=!0,G=(Q=(h=P.outputs)==null?void 0:h.values)==null?void 0:Q[b]);const C=G?$?G.name.replace("$1",V):G==null?void 0:G.name:V,B=G&&G.toBlock&&k.blocks.find(b=>b.className===G.toBlock)||k.blocks[0];let K=B.dataOutputs.find(b=>b.name===C);K||(K=this._createNewSocketConnection(C,!0),B.dataOutputs.push(K)),q.connectedPointIds.push(K.uniqueId),K.connectedPointIds.push(q.uniqueId)}else throw w.Error(["Invalid value for value connection",y]),new Error("Error parsing node connections")}if(F.flowGraphMapping.interBlockConnectors)for(const d of F.flowGraphMapping.interBlockConnectors){const y=d.input,_=d.output,x=d.isVariable;this._connectFlowGraphNodes(y,_,T.blocks[d.inputBlockIndex],T.blocks[d.outputBlockIndex],x)}if(F.flowGraphMapping.extraProcessor){const d=(et=this._interactivityGraph.declarations)==null?void 0:et[I.declaration];if(!d)throw w.Error(["No declaration found for extra processor",I]),new Error("Error parsing node connections");T.blocks=F.flowGraphMapping.extraProcessor(I,d,F.flowGraphMapping,this,T.blocks,t,this._gltf)}}}_createNewSocketConnection(t,e){return{uniqueId:D(),name:t,_connectionType:e?1:0,connectedPointIds:[]}}_connectFlowGraphNodes(t,e,o,n,s){const i=s?o.dataInputs:o.signalInputs,c=s?n.dataOutputs:n.signalOutputs,l=i.find(m=>m.name===t)||this._createNewSocketConnection(t),u=c.find(m=>m.name===e)||this._createNewSocketConnection(e,!0);i.find(m=>m.name===t)||i.push(l),c.find(m=>m.name===e)||c.push(u),l.connectedPointIds.push(u.uniqueId),u.connectedPointIds.push(l.uniqueId)}getVariableName(t){return"staticVariable_"+t}serializeToFlowGraph(){const t={uniqueId:D(),_userVariables:{},_connectionValues:{}};this._parseNodeConnections(t);for(let o=0;o<this._staticVariables.length;o++){const n=this._staticVariables[o];t._userVariables[this.getVariableName(o)]=n}return{rightHanded:!0,allBlocks:this._nodes.reduce((o,n)=>o.concat(n.blocks),[]),executionContexts:[t]}}}const W="KHR_interactivity";class ft{constructor(t){this._loader=t,this.name=W,this.enabled=this._loader.isExtensionUsed(W),this._pathConverter=Dt(this._loader.gltf),t._skipStartAnimationStep=!0;const e=t.babylonScene;e&&yt(e)}dispose(){this._loader=null,delete this._pathConverter}async onReady(){var s;if(!this._loader.babylonScene||!this._pathConverter)return;const t=this._loader.babylonScene,e=(s=this._loader.gltf.extensions)==null?void 0:s.KHR_interactivity;if(!e)return;const o=new E({scene:t});o.dispatchEventsSynchronously=!1;const n=e.graphs.map(i=>new ae(i,this._loader.gltf,this._loader.parent.targetFps).serializeToFlowGraph());await Promise.all(n.map(async i=>await jt(i,{coordinator:o,pathConverter:this._pathConverter}))),o.start()}}function yt(a){O("/extensions/KHR_interactivity/?/activeCamera/rotation",{get:()=>{if(!a.activeCamera)return new X(NaN,NaN,NaN,NaN);const t=X.FromRotationMatrix(a.activeCamera.getWorldMatrix()).normalize();return a.useRightHandedSystem||(t.w*=-1,t.x*=-1),t},type:"Quaternion",getTarget:()=>a.activeCamera}),O("/extensions/KHR_interactivity/?/activeCamera/position",{get:()=>{if(!a.activeCamera)return new ut(NaN,NaN,NaN);const t=a.activeCamera.getWorldMatrix().getTranslation();return a.useRightHandedSystem||(t.x*=-1),t},type:"Vector3",getTarget:()=>a.activeCamera}),O("/animations/{}/extensions/KHR_interactivity/isPlaying",{get:t=>{var e;return((e=t._babylonAnimationGroup)==null?void 0:e.isPlaying)??!1},type:"boolean",getTarget:t=>t._babylonAnimationGroup}),O("/animations/{}/extensions/KHR_interactivity/minTime",{get:t=>{var e;return(((e=t._babylonAnimationGroup)==null?void 0:e.from)??0)/60},type:"number",getTarget:t=>t._babylonAnimationGroup}),O("/animations/{}/extensions/KHR_interactivity/maxTime",{get:t=>{var e;return(((e=t._babylonAnimationGroup)==null?void 0:e.to)??0)/60},type:"number",getTarget:t=>t._babylonAnimationGroup}),O("/animations/{}/extensions/KHR_interactivity/playhead",{get:t=>{var e;return(((e=t._babylonAnimationGroup)==null?void 0:e.getCurrentFrame())??0)/60},type:"number",getTarget:t=>t._babylonAnimationGroup}),O("/animations/{}/extensions/KHR_interactivity/virtualPlayhead",{get:t=>{var e;return(((e=t._babylonAnimationGroup)==null?void 0:e.getCurrentFrame())??0)/60},type:"number",getTarget:t=>t._babylonAnimationGroup})}Qt(W,"FlowGraphGLTFDataProvider",async()=>(await r(async()=>{const{FlowGraphGLTFDataProvider:a}=await import("./flowGraphGLTFDataProvider-By82y__G.js");return{FlowGraphGLTFDataProvider:a}},__vite__mapDeps([84,1,2,3,4,5,23,24,9,21,25,26,13,12,14,15,16,17,18,19,20,22,27,28,8,29,30,31,10]))).FlowGraphGLTFDataProvider);Ot(W);xt(W,!0,a=>new ft(a));const ke=Object.freeze(Object.defineProperty({__proto__:null,KHR_interactivity:ft,_AddInteractivityObjectModel:yt},Symbol.toStringTag,{value:"Module"}));export{$t as F,Mt as G,ke as K,dt as _,Zt as a,Jt as b,Xt as c,E as d,de as e,fe as f,Wt as g,ye as h,me as i,ge as j,J as k,tt as l};
|
trackio/frontend/dist/assets/KHR_lights_punctual-CCzRzwty.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{M as x,V as h}from"./math.vector-C1xyPDEl.js";import{Color3 as b}from"./math.color-CzLaKyb6.js";import{_ as d,s as u}from"./bufferUtils-DTRdUD8L.js";import{N as T}from"./node-C-Iu0ALa.js";import{L as S}from"./light-DT53MGnW.js";import{b as E,c as v}from"./objectModelMapping-DWcnlpHl.js";import{R as Z}from"./math.scalar.functions-DkeaWAWz.js";import{A as D,G as M}from"./glTFLoader-BPeiodqB.js";import{u as C,r as R}from"./dynamic-BUKysMyT.js";import"./decorators.serialization-C7d7ykGG.js";import"./lightConstants-B1TSTh-k.js";import"./tools-D01N7ptR.js";import"./math.axis-Ck7JfPPj.js";import"./texture-BEqhDl5Y.js";import"./baseTexture-CgAggsey.js";import"./math.plane-Ce_75-g5.js";import"./index-Bx1QoKVU.js";import"./mesh-Fj-9677A.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./material-CrEsrP1R.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";T.AddNodeConstructor("Light_Type_1",(m,t)=>()=>new a(m,h.Zero(),t));class a extends E{get shadowFrustumSize(){return this._shadowFrustumSize}set shadowFrustumSize(t){this._shadowFrustumSize=t,this.forceProjectionMatrixCompute()}get shadowOrthoScale(){return this._shadowOrthoScale}set shadowOrthoScale(t){this._shadowOrthoScale=t,this.forceProjectionMatrixCompute()}get orthoLeft(){return this._orthoLeft}set orthoLeft(t){this._orthoLeft=t}get orthoRight(){return this._orthoRight}set orthoRight(t){this._orthoRight=t}get orthoTop(){return this._orthoTop}set orthoTop(t){this._orthoTop=t}get orthoBottom(){return this._orthoBottom}set orthoBottom(t){this._orthoBottom=t}constructor(t,o,e,i){super(t,e,i),this._shadowFrustumSize=0,this._shadowOrthoScale=.1,this.autoUpdateExtends=!0,this.autoCalcShadowZBounds=!1,this._orthoLeft=Number.MAX_VALUE,this._orthoRight=Number.MIN_VALUE,this._orthoTop=Number.MIN_VALUE,this._orthoBottom=Number.MAX_VALUE,this.position=o.scale(-1),this.direction=o}getClassName(){return"DirectionalLight"}getTypeID(){return S.LIGHTTYPEID_DIRECTIONALLIGHT}_setDefaultShadowProjectionMatrix(t,o,e){this.shadowFrustumSize>0?this._setDefaultFixedFrustumShadowProjectionMatrix(t):this._setDefaultAutoExtendShadowProjectionMatrix(t,o,e)}_setDefaultFixedFrustumShadowProjectionMatrix(t){const o=this.getScene().activeCamera;x.OrthoLHToRef(this.shadowFrustumSize,this.shadowFrustumSize,this.shadowMinZ!==void 0?this.shadowMinZ:o?o.minZ:0,this.shadowMaxZ!==void 0?this.shadowMaxZ:o?o.maxZ:1e4,t,this.getScene().getEngine().isNDCHalfZRange)}_setDefaultAutoExtendShadowProjectionMatrix(t,o,e){const i=this.getScene().activeCamera;if(this.autoUpdateExtends||this._orthoLeft===Number.MAX_VALUE){const r=h.Zero();this._orthoLeft=Number.MAX_VALUE,this._orthoRight=-Number.MAX_VALUE,this._orthoTop=-Number.MAX_VALUE,this._orthoBottom=Number.MAX_VALUE;let g=Number.MAX_VALUE,L=-Number.MAX_VALUE;for(let w=0;w<e.length;w++){const A=e[w];if(!A)continue;const B=A.getBoundingInfo().boundingBox;for(let y=0;y<B.vectorsWorld.length;y++)h.TransformCoordinatesToRef(B.vectorsWorld[y],o,r),r.x<this._orthoLeft&&(this._orthoLeft=r.x),r.y<this._orthoBottom&&(this._orthoBottom=r.y),r.x>this._orthoRight&&(this._orthoRight=r.x),r.y>this._orthoTop&&(this._orthoTop=r.y),this.autoCalcShadowZBounds&&(r.z<g&&(g=r.z),r.z>L&&(L=r.z))}this.autoCalcShadowZBounds&&(this._shadowMinZ=g,this._shadowMaxZ=L)}const c=this._orthoRight-this._orthoLeft,f=this._orthoTop-this._orthoBottom,s=this.shadowMinZ!==void 0?this.shadowMinZ:(i==null?void 0:i.minZ)||0,n=this.shadowMaxZ!==void 0?this.shadowMaxZ:(i==null?void 0:i.maxZ)||1e4,l=this.getScene().getEngine().useReverseDepthBuffer;x.OrthoOffCenterLHToRef(this._orthoLeft-c*this.shadowOrthoScale,this._orthoRight+c*this.shadowOrthoScale,this._orthoBottom-f*this.shadowOrthoScale,this._orthoTop+f*this.shadowOrthoScale,l?n:s,l?s:n,t,this.getScene().getEngine().isNDCHalfZRange)}_buildUniformLayout(){this._uniformBuffer.addUniform("vLightData",4),this._uniformBuffer.addUniform("vLightDiffuse",4),this._uniformBuffer.addUniform("vLightSpecular",4),this._uniformBuffer.addUniform("shadowsInfo",3),this._uniformBuffer.addUniform("depthValues",2),this._uniformBuffer.create()}transferToEffect(t,o){return this.computeTransformedInformation()?(this._uniformBuffer.updateFloat4("vLightData",this.transformedDirection.x,this.transformedDirection.y,this.transformedDirection.z,1,o),this):(this._uniformBuffer.updateFloat4("vLightData",this.direction.x,this.direction.y,this.direction.z,1,o),this)}transferToNodeMaterialEffect(t,o){return this.computeTransformedInformation()?(t.setFloat3(o,this.transformedDirection.x,this.transformedDirection.y,this.transformedDirection.z),this):(t.setFloat3(o,this.direction.x,this.direction.y,this.direction.z),this)}getDepthMinZ(t){const o=this._scene.getEngine();return!o.useReverseDepthBuffer&&o.isNDCHalfZRange?0:1}getDepthMaxZ(t){const o=this._scene.getEngine();return o.useReverseDepthBuffer&&o.isNDCHalfZRange?0:1}prepareLightSpecificDefines(t,o){t["DIRLIGHT"+o]=!0}}d([u()],a.prototype,"shadowFrustumSize",null);d([u()],a.prototype,"shadowOrthoScale",null);d([u()],a.prototype,"autoUpdateExtends",void 0);d([u()],a.prototype,"autoCalcShadowZBounds",void 0);d([u("orthoLeft")],a.prototype,"_orthoLeft",void 0);d([u("orthoRight")],a.prototype,"_orthoRight",void 0);d([u("orthoTop")],a.prototype,"_orthoTop",void 0);d([u("orthoBottom")],a.prototype,"_orthoBottom",void 0);Z("BABYLON.DirectionalLight",a);T.AddNodeConstructor("Light_Type_0",(m,t)=>()=>new _(m,h.Zero(),t));class _ extends E{get shadowAngle(){return this._shadowAngle}set shadowAngle(t){this._shadowAngle=t,this.forceProjectionMatrixCompute()}get direction(){return this._direction}set direction(t){const o=this.needCube();if(this._direction=t,this.needCube()!==o&&this._shadowGenerators){const e=this._shadowGenerators.values();for(let i=e.next();i.done!==!0;i=e.next())i.value.recreateShadowMap()}}constructor(t,o,e,i){super(t,e,i),this._shadowAngle=Math.PI/2,this.position=o}getClassName(){return"PointLight"}getTypeID(){return S.LIGHTTYPEID_POINTLIGHT}needCube(){return!this.direction}getShadowDirection(t){if(this.direction)return super.getShadowDirection(t);switch(t){case 0:return new h(1,0,0);case 1:return new h(-1,0,0);case 2:return new h(0,-1,0);case 3:return new h(0,1,0);case 4:return new h(0,0,1);case 5:return new h(0,0,-1)}return h.Zero()}_setDefaultShadowProjectionMatrix(t,o,e){const i=this.getScene().activeCamera,c=this.getDepthMinZ(i),f=this.getDepthMaxZ(i),s=this.getScene().getEngine().useReverseDepthBuffer;x.PerspectiveFovLHToRef(this.shadowAngle,1,s?f:c,s?c:f,t,!0,this._scene.getEngine().isNDCHalfZRange,void 0,s)}_buildUniformLayout(){this._uniformBuffer.addUniform("vLightData",4),this._uniformBuffer.addUniform("vLightDiffuse",4),this._uniformBuffer.addUniform("vLightSpecular",4),this._uniformBuffer.addUniform("vLightFalloff",4),this._uniformBuffer.addUniform("shadowsInfo",3),this._uniformBuffer.addUniform("depthValues",2),this._uniformBuffer.create()}transferToEffect(t,o){const e=this._scene.floatingOriginOffset;return this.computeTransformedInformation()?this._uniformBuffer.updateFloat4("vLightData",this.transformedPosition.x-e.x,this.transformedPosition.y-e.y,this.transformedPosition.z-e.z,0,o):this._uniformBuffer.updateFloat4("vLightData",this.position.x-e.x,this.position.y-e.y,this.position.z-e.z,0,o),this._uniformBuffer.updateFloat4("vLightFalloff",this.range,this._inverseSquaredRange,0,0,o),this}transferToNodeMaterialEffect(t,o){const e=this._scene.floatingOriginOffset;return this.computeTransformedInformation()?t.setFloat3(o,this.transformedPosition.x-e.x,this.transformedPosition.y-e.y,this.transformedPosition.z-e.z):t.setFloat3(o,this.position.x-e.x,this.position.y-e.y,this.position.z-e.z),this}prepareLightSpecificDefines(t,o){t["POINTLIGHT"+o]=!0}}d([u()],_.prototype,"shadowAngle",null);Z("BABYLON.PointLight",_);const p="KHR_lights_punctual";class N{constructor(t){this.name=p,this._loader=t,this.enabled=this._loader.isExtensionUsed(p)}dispose(){this._loader=null,delete this._lights}onLoading(){const t=this._loader.gltf.extensions;if(t&&t[this.name]){const o=t[this.name];this._lights=o.lights,D.Assign(this._lights)}}loadNodeAsync(t,o,e){return M.LoadExtensionAsync(t,o,this.name,async(i,c)=>(this._loader._allMaterialsDirtyRequired=!0,await this._loader.loadNodeAsync(t,o,f=>{let s;const n=D.Get(i,this._lights,c.light),l=n.name||f.name;switch(this._loader.babylonScene._blockEntityCollection=!!this._loader._assetContainer,n.type){case"directional":{const r=new a(l,h.Backward(),this._loader.babylonScene);r.position.setAll(0),s=r;break}case"point":{s=new _(l,h.Zero(),this._loader.babylonScene);break}case"spot":{const r=new v(l,h.Zero(),h.Backward(),0,1,this._loader.babylonScene);r.angle=(n.spot&&n.spot.outerConeAngle||Math.PI/4)*2,r.innerAngle=(n.spot&&n.spot.innerConeAngle||0)*2,s=r;break}default:throw this._loader.babylonScene._blockEntityCollection=!1,new Error(`${i}: Invalid light type (${n.type})`)}s._parentContainer=this._loader._assetContainer,this._loader.babylonScene._blockEntityCollection=!1,n._babylonLight=s,s.falloffType=S.FALLOFF_GLTF,s.diffuse=n.color?b.FromArray(n.color):b.White(),s.intensity=n.intensity==null?1:n.intensity,s.range=n.range==null?Number.MAX_VALUE:n.range,s.parent=f,this._loader._babylonLights.push(s),M.AddPointerMetadata(s,i),e(f)})))}}C(p);R(p,!0,m=>new N(m));export{N as KHR_lights};
|
trackio/frontend/dist/assets/KHR_materials_anisotropy-DeuhwPRx.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{G as l}from"./glTFLoader-BPeiodqB.js";import{u as d,r as c}from"./dynamic-BUKysMyT.js";import"./index-Bx1QoKVU.js";import"./math.vector-C1xyPDEl.js";import"./math.scalar.functions-DkeaWAWz.js";import"./math.color-CzLaKyb6.js";import"./tools-D01N7ptR.js";import"./decorators.serialization-C7d7ykGG.js";import"./bufferUtils-DTRdUD8L.js";import"./mesh-Fj-9677A.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./node-C-Iu0ALa.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./math.axis-Ck7JfPPj.js";import"./material-CrEsrP1R.js";import"./texture-BEqhDl5Y.js";import"./baseTexture-CgAggsey.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";import"./light-DT53MGnW.js";import"./objectModelMapping-DWcnlpHl.js";const n="KHR_materials_anisotropy";class u{constructor(t){this.name=n,this.order=195,this._loader=t,this.enabled=this._loader.isExtensionUsed(n)}dispose(){this._loader=null}loadMaterialPropertiesAsync(t,o,i){return l.LoadExtensionAsync(t,o,this.name,async(r,e)=>{const s=new Array;s.push(this._loader.loadMaterialPropertiesAsync(t,o,i)),s.push(this._loadAnisotropyPropertiesAsync(r,e,i)),await Promise.all(s)})}async _loadAnisotropyPropertiesAsync(t,o,i){const r=this._loader._getOrCreateMaterialAdapter(i),e=new Array,s=o.anisotropyStrength??0,y=o.anisotropyRotation??0;r.specularRoughnessAnisotropy=s,r.geometryTangentAngle=y;const a=o.extensions??{};(!a.EXT_materials_anisotropy_openpbr||!a.EXT_materials_anisotropy_openpbr.openPbrAnisotropyEnabled)&&r.configureGltfStyleAnisotropy(!0),o.anisotropyTexture&&(o.anisotropyTexture.nonColorData=!0,e.push(this._loader.loadTextureInfoAsync(`${t}/anisotropyTexture`,o.anisotropyTexture,m=>{m.name=`${i.name} (Anisotropy Intensity)`,r.geometryTangentTexture=m}))),await Promise.all(e)}}d(n);c(n,!0,p=>new u(p));export{u as KHR_materials_anisotropy};
|
trackio/frontend/dist/assets/KHR_materials_clearcoat-CqejipXp.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{G as n}from"./glTFLoader-BPeiodqB.js";import{u as l,r as u}from"./dynamic-BUKysMyT.js";import"./index-Bx1QoKVU.js";import"./math.vector-C1xyPDEl.js";import"./math.scalar.functions-DkeaWAWz.js";import"./math.color-CzLaKyb6.js";import"./tools-D01N7ptR.js";import"./decorators.serialization-C7d7ykGG.js";import"./bufferUtils-DTRdUD8L.js";import"./mesh-Fj-9677A.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./node-C-Iu0ALa.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./math.axis-Ck7JfPPj.js";import"./material-CrEsrP1R.js";import"./texture-BEqhDl5Y.js";import"./baseTexture-CgAggsey.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";import"./light-DT53MGnW.js";import"./objectModelMapping-DWcnlpHl.js";const m="KHR_materials_clearcoat";class d{constructor(a){this.name=m,this.order=190,this._loader=a,this.enabled=this._loader.isExtensionUsed(m)}dispose(){this._loader=null}loadMaterialPropertiesAsync(a,o,r){return n.LoadExtensionAsync(a,o,this.name,async(t,s)=>{const e=new Array;e.push(this._loader.loadMaterialPropertiesAsync(a,o,r)),e.push(this._loadClearCoatPropertiesAsync(t,s,r)),await Promise.all(e)})}_loadClearCoatPropertiesAsync(a,o,r){const t=this._loader._getOrCreateMaterialAdapter(r),s=new Array;return t.configureCoat(),t.coatWeight=o.clearcoatFactor!==void 0?o.clearcoatFactor:0,t.coatRoughness=o.clearcoatRoughnessFactor!==void 0?o.clearcoatRoughnessFactor:0,o.clearcoatTexture&&s.push(this._loader.loadTextureInfoAsync(`${a}/clearcoatTexture`,o.clearcoatTexture,e=>{e.name=`${r.name} (ClearCoat)`,t.coatWeightTexture=e})),o.clearcoatRoughnessTexture&&(o.clearcoatRoughnessTexture.nonColorData=!0,s.push(this._loader.loadTextureInfoAsync(`${a}/clearcoatRoughnessTexture`,o.clearcoatRoughnessTexture,e=>{e.name=`${r.name} (ClearCoat Roughness)`,t.coatRoughnessTexture=e}))),o.clearcoatNormalTexture&&(o.clearcoatNormalTexture.nonColorData=!0,s.push(this._loader.loadTextureInfoAsync(`${a}/clearcoatNormalTexture`,o.clearcoatNormalTexture,e=>{var c;e.name=`${r.name} (ClearCoat Normal)`,t.geometryCoatNormalTexture=e,((c=o.clearcoatNormalTexture)==null?void 0:c.scale)!=null&&(t.geometryCoatNormalTextureScale=o.clearcoatNormalTexture.scale)})),t.setNormalMapInversions(!r.getScene().useRightHandedSystem,r.getScene().useRightHandedSystem)),Promise.all(s).then(()=>{})}}l(m);u(m,!0,i=>new d(i));export{d as KHR_materials_clearcoat};
|
trackio/frontend/dist/assets/KHR_materials_coat-DOoUAE9E.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
import{G as d}from"./glTFLoader-BPeiodqB.js";import{u as h,r as T}from"./dynamic-BUKysMyT.js";import{Color3 as g}from"./math.color-CzLaKyb6.js";import"./index-Bx1QoKVU.js";import"./math.vector-C1xyPDEl.js";import"./math.scalar.functions-DkeaWAWz.js";import"./tools-D01N7ptR.js";import"./decorators.serialization-C7d7ykGG.js";import"./bufferUtils-DTRdUD8L.js";import"./mesh-Fj-9677A.js";import"./mesh.vertexData-Cr3zcPIw.js";import"./buffer-DqKlo1ts.js";import"./dataBuffer-CgoqiUzT.js";import"./subMesh-UWDo5wjI.js";import"./drawWrapper-CZVk0xN-.js";import"./smartArray-CO6iJ0u5.js";import"./node-C-Iu0ALa.js";import"./math.viewport-CgkTt1RS.js";import"./sceneComponent-Dflhczui.js";import"./math.plane-Ce_75-g5.js";import"./lightConstants-B1TSTh-k.js";import"./sceneLoaderFlags-wywyb9ml.js";import"./math.axis-Ck7JfPPj.js";import"./material-CrEsrP1R.js";import"./texture-BEqhDl5Y.js";import"./baseTexture-CgAggsey.js";import"./drawWrapper.functions-Dzz0ghog.js";import"./cameraInputsManager-WrTH9PVf.js";import"./deviceInputEvents-BTza-zoR.js";import"./bone-B35odufe.js";import"./skeleton-BVUVDN0a.js";import"./rawTexture-Cp8_XuDs.js";import"./animation-IftAdLIx.js";import"./sceneLoader-CutT-qCn.js";import"./scene-gn_hoGMy.js";import"./renderingManager-CiY2OUrf.js";import"./perfCounter-DusmGz8t.js";import"./assetContainer-DOnbxwKX.js";import"./instancedMesh-Cz-e1Hca.js";import"./light-DT53MGnW.js";import"./objectModelMapping-DWcnlpHl.js";const i="KHR_materials_coat";class f{constructor(r){this.name=i,this.order=191,this.useOpenPBR=!1,this._loader=r,this.enabled=this._loader.isExtensionUsed(i)}dispose(){this._loader=null}loadMaterialPropertiesAsync(r,o,a){return d.LoadExtensionAsync(r,o,this.name,async(t,n)=>{const s=new Array;s.push(this._loader.loadMaterialPropertiesAsync(r,o,a)),o.extensions&&o.extensions.KHR_materials_openpbr&&(this.useOpenPBR=!0),s.push(this._loadCoatPropertiesAsync(t,n,a)),await Promise.all(s)})}_loadCoatPropertiesAsync(r,o,a){const t=this._loader._getOrCreateMaterialAdapter(a),n=new Array;t.configureCoat(),t.coatWeight=o.coatFactor!==void 0?o.coatFactor:0,t.coatRoughness=o.coatRoughnessFactor!==void 0?o.coatRoughnessFactor:0,o.coatTexture&&n.push(this._loader.loadTextureInfoAsync(`${r}/coatTexture`,o.coatTexture,e=>{e.name=`${a.name} (Coat)`,t.coatWeightTexture=e})),o.coatRoughnessTexture&&(o.coatRoughnessTexture.nonColorData=!0,n.push(this._loader.loadTextureInfoAsync(`${r}/coatRoughnessTexture`,o.coatRoughnessTexture,e=>{e.name=`${a.name} (Coat Roughness)`,t.coatRoughnessTexture=e}))),o.coatNormalTexture&&(o.coatNormalTexture.nonColorData=!0,n.push(this._loader.loadTextureInfoAsync(`${r}/coatNormalTexture`,o.coatNormalTexture,e=>{var m;e.name=`${a.name} (Coat Normal)`,t.geometryCoatNormalTexture=e,((m=o.coatNormalTexture)==null?void 0:m.scale)!=null&&(t.geometryCoatNormalTextureScale=o.coatNormalTexture.scale)})),t.setNormalMapInversions(!a.getScene().useRightHandedSystem,a.getScene().useRightHandedSystem)),t.coatDarkening=o.coatDarkeningFactor!==void 0?o.coatDarkeningFactor:1,t.coatIor=o.coatIor!==void 0?o.coatIor:1.5;const s=g.White();o.coatColorFactor!==void 0&&s.fromArray(o.coatColorFactor),t.coatColor=s,o.coatColorTexture&&n.push(this._loader.loadTextureInfoAsync(`${r}/coatColorTexture`,o.coatColorTexture,e=>{e.name=`${a.name} (Coat Color)`,t.coatColorTexture=e}));const u=o.coatAnisotropyStrength??0,l=o.coatAnisotropyRotation??0;return t.coatRoughnessAnisotropy=u,t.geometryCoatTangentAngle=l,this.useOpenPBR||t.configureGltfStyleAnisotropy(!0),o.coatAnisotropyTexture&&(o.coatAnisotropyTexture.nonColorData=!0,n.push(this._loader.loadTextureInfoAsync(`${r}/coatAnisotropyTexture`,o.coatAnisotropyTexture,e=>{e.name=`${a.name} (Coat Anisotropy)`,t.geometryCoatTangentTexture=e}))),Promise.all(n).then(()=>{})}}h(i);T(i,!0,c=>new f(c));export{f as KHR_materials_coat};
|