Update v1.1 publications and Vons branding
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +2 -2
- .github/workflows/checks.yml +6 -0
- .gitignore +12 -0
- README.md +14 -6
- configs/pilot-v2-diffusion.json +28 -0
- configs/pilot-v2-direct.json +21 -0
- configs/public-release.json +39 -1
- docs/COMMUNITY.md +4 -3
- docs/HUB_PUBLISHING.md +23 -14
- docs/HUGGING_FACE.md +14 -6
- docs/MODEL_CARD.md +11 -5
- docs/PUBLIC_RELEASE.md +16 -6
- docs/assets/vons-extension-icon-128.png +0 -0
- docs/assets/vons-extension-icon-16.png +0 -0
- docs/assets/vons-extension-icon-32.png +0 -0
- docs/assets/vons-extension-icon-48.png +0 -0
- docs/assets/vons-logo.png +0 -0
- docs/publications/README.md +29 -13
- docs/publications/vons-paper-v1.1.pdf +0 -0
- docs/publications/vons-tech-report-v1.1.pdf +0 -0
- sdk/typescript/README.md +5 -0
- sdk/typescript/extension/ONNX_RUNTIME_NOTICES.md +0 -0
- sdk/typescript/extension/README.md +106 -0
- sdk/typescript/extension/background.mjs +4 -0
- sdk/typescript/extension/build.mjs +51 -0
- sdk/typescript/extension/bundle.ts +100 -0
- sdk/typescript/extension/manifest.json +20 -0
- sdk/typescript/extension/panel.css +61 -0
- sdk/typescript/extension/panel.html +44 -0
- sdk/typescript/extension/panel.ts +146 -0
- sdk/typescript/extension/privacy.html +10 -0
- sdk/typescript/extension/storage.ts +23 -0
- sdk/typescript/extension/tsconfig.json +13 -0
- sdk/typescript/extension/worker.ts +42 -0
- sdk/typescript/package-lock.json +18 -3
- sdk/typescript/package.json +6 -3
- sdk/typescript/src/onnx.ts +23 -11
- sdk/typescript/test/extension.test.mjs +46 -0
- sdk/typescript/test/onnx.test.mjs +56 -1
- tests/test_calibration.py +90 -6
- tests/test_data.py +20 -1
- tests/test_diffusion_model.py +90 -0
- tests/test_inference.py +323 -0
- tests/test_mind2web_adapter.py +101 -0
- tests/test_mind2web_prediction.py +1 -0
- tests/test_onnx_export.py +374 -0
- tools/index_mind2web_predictions.py +110 -0
- tools/predict_mind2web.py +34 -13
- tools/prepare_public_release.py +1 -1
- tools/replay_mind2web_evaluation.py +1 -0
.gitattributes
CHANGED
|
@@ -8,5 +8,5 @@
|
|
| 8 |
*.cff text eol=lf
|
| 9 |
*.pdf binary
|
| 10 |
*.png binary
|
| 11 |
-
docs/publications/vons-paper-v1.
|
| 12 |
-
docs/publications/vons-tech-report-v1.
|
|
|
|
| 8 |
*.cff text eol=lf
|
| 9 |
*.pdf binary
|
| 10 |
*.png binary
|
| 11 |
+
docs/publications/vons-paper-v1.1.pdf filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
docs/publications/vons-tech-report-v1.1.pdf filter=lfs diff=lfs merge=lfs -text
|
.github/workflows/checks.yml
CHANGED
|
@@ -9,6 +9,8 @@ on:
|
|
| 9 |
- 'tests/**'
|
| 10 |
- 'configs/**'
|
| 11 |
- 'sdk/**'
|
|
|
|
|
|
|
| 12 |
- 'pyproject.toml'
|
| 13 |
- '.gitignore'
|
| 14 |
pull_request:
|
|
@@ -19,6 +21,8 @@ on:
|
|
| 19 |
- 'tests/**'
|
| 20 |
- 'configs/**'
|
| 21 |
- 'sdk/**'
|
|
|
|
|
|
|
| 22 |
- 'pyproject.toml'
|
| 23 |
- '.gitignore'
|
| 24 |
workflow_dispatch:
|
|
@@ -59,3 +63,5 @@ jobs:
|
|
| 59 |
- run: npm run build:browser
|
| 60 |
- run: npm run build:benchmark
|
| 61 |
- run: npm run build:parity
|
|
|
|
|
|
|
|
|
| 9 |
- 'tests/**'
|
| 10 |
- 'configs/**'
|
| 11 |
- 'sdk/**'
|
| 12 |
+
- 'docs/assets/inlevel9-signature.png'
|
| 13 |
+
- 'docs/assets/vons-extension-icon-*.png'
|
| 14 |
- 'pyproject.toml'
|
| 15 |
- '.gitignore'
|
| 16 |
pull_request:
|
|
|
|
| 21 |
- 'tests/**'
|
| 22 |
- 'configs/**'
|
| 23 |
- 'sdk/**'
|
| 24 |
+
- 'docs/assets/inlevel9-signature.png'
|
| 25 |
+
- 'docs/assets/vons-extension-icon-*.png'
|
| 26 |
- 'pyproject.toml'
|
| 27 |
- '.gitignore'
|
| 28 |
workflow_dispatch:
|
|
|
|
| 63 |
- run: npm run build:browser
|
| 64 |
- run: npm run build:benchmark
|
| 65 |
- run: npm run build:parity
|
| 66 |
+
- run: npm run typecheck:extension
|
| 67 |
+
- run: npm run build:extension
|
.gitignore
CHANGED
|
@@ -31,12 +31,21 @@
|
|
| 31 |
!/docs/publications/README.md
|
| 32 |
!/docs/publications/LICENSE.md
|
| 33 |
!/docs/publications/ARXIV.md
|
|
|
|
|
|
|
| 34 |
/docs/assets/*
|
| 35 |
!/docs/assets/inlevel9-signature.png
|
| 36 |
!/docs/assets/inlevel9-signature-dark.png
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
/configs/*
|
| 38 |
!/configs/pilot.json
|
| 39 |
!/configs/pilot-diffusion.json
|
|
|
|
|
|
|
| 40 |
!/configs/public-release.json
|
| 41 |
/sdk/*
|
| 42 |
!/sdk/typescript/
|
|
@@ -48,6 +57,7 @@
|
|
| 48 |
!/sdk/typescript/src/
|
| 49 |
!/sdk/typescript/test/
|
| 50 |
!/sdk/typescript/demo/
|
|
|
|
| 51 |
|
| 52 |
# Raw/generated data, weights, evidence packets and local publications.
|
| 53 |
data/
|
|
@@ -89,6 +99,8 @@ restricted/
|
|
| 89 |
# Only the exact reviewed publication bytes are eligible for staging.
|
| 90 |
!/docs/publications/vons-paper-v1.0.pdf
|
| 91 |
!/docs/publications/vons-tech-report-v1.0.1.pdf
|
|
|
|
|
|
|
| 92 |
|
| 93 |
# Credentials, machine-specific config, private prompts and service caches.
|
| 94 |
.env
|
|
|
|
| 31 |
!/docs/publications/README.md
|
| 32 |
!/docs/publications/LICENSE.md
|
| 33 |
!/docs/publications/ARXIV.md
|
| 34 |
+
!/docs/publications/vons-paper-v1.1.pdf
|
| 35 |
+
!/docs/publications/vons-tech-report-v1.1.pdf
|
| 36 |
/docs/assets/*
|
| 37 |
!/docs/assets/inlevel9-signature.png
|
| 38 |
!/docs/assets/inlevel9-signature-dark.png
|
| 39 |
+
!/docs/assets/vons-logo.png
|
| 40 |
+
!/docs/assets/vons-extension-icon-16.png
|
| 41 |
+
!/docs/assets/vons-extension-icon-32.png
|
| 42 |
+
!/docs/assets/vons-extension-icon-48.png
|
| 43 |
+
!/docs/assets/vons-extension-icon-128.png
|
| 44 |
/configs/*
|
| 45 |
!/configs/pilot.json
|
| 46 |
!/configs/pilot-diffusion.json
|
| 47 |
+
!/configs/pilot-v2-direct.json
|
| 48 |
+
!/configs/pilot-v2-diffusion.json
|
| 49 |
!/configs/public-release.json
|
| 50 |
/sdk/*
|
| 51 |
!/sdk/typescript/
|
|
|
|
| 57 |
!/sdk/typescript/src/
|
| 58 |
!/sdk/typescript/test/
|
| 59 |
!/sdk/typescript/demo/
|
| 60 |
+
!/sdk/typescript/extension/
|
| 61 |
|
| 62 |
# Raw/generated data, weights, evidence packets and local publications.
|
| 63 |
data/
|
|
|
|
| 99 |
# Only the exact reviewed publication bytes are eligible for staging.
|
| 100 |
!/docs/publications/vons-paper-v1.0.pdf
|
| 101 |
!/docs/publications/vons-tech-report-v1.0.1.pdf
|
| 102 |
+
!/docs/publications/vons-paper-v1.1.pdf
|
| 103 |
+
!/docs/publications/vons-tech-report-v1.1.pdf
|
| 104 |
|
| 105 |
# Credentials, machine-specific config, private prompts and service caches.
|
| 106 |
.env
|
README.md
CHANGED
|
@@ -11,6 +11,8 @@ tags:
|
|
| 11 |
- onnx
|
| 12 |
---
|
| 13 |
|
|
|
|
|
|
|
| 14 |

|
| 15 |
|
| 16 |
# Vons
|
|
@@ -57,17 +59,23 @@ an organization card, source preview and later a separately validated Space.
|
|
| 57 |
|
| 58 |
| Publication | Download |
|
| 59 |
| --- | --- |
|
| 60 |
-
| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows —
|
| 61 |
-
| Vons: Compact Decision Models for Frontier-Agent Workflows — Technical Report v1.
|
| 62 |
|
| 63 |
Articles and original figures use **CC BY 4.0**. Their version, SHA-256 digests
|
| 64 |
and verification scope are in the [publication index](docs/publications/README.md).
|
| 65 |
-
These preprints preserve negative results
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
|
|
|
| 69 |
acceptance is claimed.
|
| 70 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
## Local use
|
| 72 |
|
| 73 |
```sh
|
|
|
|
| 11 |
- onnx
|
| 12 |
---
|
| 13 |
|
| 14 |
+

|
| 15 |
+
|
| 16 |

|
| 17 |
|
| 18 |
# Vons
|
|
|
|
| 59 |
|
| 60 |
| Publication | Download |
|
| 61 |
| --- | --- |
|
| 62 |
+
| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows — Preprint v1.1 | [PDF · 21 pages](docs/publications/vons-paper-v1.1.pdf) |
|
| 63 |
+
| Vons: Compact Decision Models for Frontier-Agent Workflows — Technical Report v1.1 | [PDF · 29 pages](docs/publications/vons-tech-report-v1.1.pdf) |
|
| 64 |
|
| 65 |
Articles and original figures use **CC BY 4.0**. Their version, SHA-256 digests
|
| 66 |
and verification scope are in the [publication index](docs/publications/README.md).
|
| 67 |
+
These preprints preserve negative results, missing measurements and the boundary
|
| 68 |
+
between frozen v1 evidence and the separately labelled post-freeze v2 handoff.
|
| 69 |
+
The paper was submitted to arXiv under **submission number 8127259**; the last
|
| 70 |
+
checked status on **2026-09-25** is **on hold** (undergoing checks). This tracking
|
| 71 |
+
number is not a public arXiv article identifier. No announcement or peer-review
|
| 72 |
acceptance is claimed.
|
| 73 |
|
| 74 |
+
Chrome Web Store version **0.1.0** passed review and is publicly available as
|
| 75 |
+
[Vons — Local Decisions](https://chromewebstore.google.com/detail/vons-%E2%80%94-local-decisions/cbmjkokoojinfgafahncmjlfmicinide),
|
| 76 |
+
verified on 2026-09-25. Store availability is not evidence of model quality,
|
| 77 |
+
scientific validity or production readiness.
|
| 78 |
+
|
| 79 |
## Local use
|
| 80 |
|
| 81 |
```sh
|
configs/pilot-v2-diffusion.json
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_id": "vons-pilot-v2-diffusion",
|
| 3 |
+
"backend": "diffusion",
|
| 4 |
+
"option_count": 32,
|
| 5 |
+
"encoder": {
|
| 6 |
+
"name": "google/bert_uncased_L-4_H-256_A-4",
|
| 7 |
+
"revision": "387825ce42dbb39b87911cdf8e383ee3b25184f8",
|
| 8 |
+
"max_tokens": 512
|
| 9 |
+
},
|
| 10 |
+
"data": {
|
| 11 |
+
"path": "data/generated/pilot-v2.jsonl",
|
| 12 |
+
"manifest": "data/generated/pilot-v2.manifest.json"
|
| 13 |
+
},
|
| 14 |
+
"seed": 7,
|
| 15 |
+
"epochs": 12,
|
| 16 |
+
"batch_size": 32,
|
| 17 |
+
"learning_rate": 0.001,
|
| 18 |
+
"latent_size": 64,
|
| 19 |
+
"attention_heads": 4,
|
| 20 |
+
"diffusion_conditioning": "token_cross_attention",
|
| 21 |
+
"noise_schedule": "cosine",
|
| 22 |
+
"diffusion_steps": 64,
|
| 23 |
+
"inference_steps": 8,
|
| 24 |
+
"soft_target_smoothing": 0.1,
|
| 25 |
+
"device": "auto",
|
| 26 |
+
"artifact_dir": "artifacts/pilot-v2-diffusion",
|
| 27 |
+
"target_bundle_bytes": 67108864
|
| 28 |
+
}
|
configs/pilot-v2-direct.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_id": "vons-pilot-v2-direct",
|
| 3 |
+
"backend": "direct",
|
| 4 |
+
"option_count": 32,
|
| 5 |
+
"encoder": {
|
| 6 |
+
"name": "google/bert_uncased_L-4_H-256_A-4",
|
| 7 |
+
"revision": "387825ce42dbb39b87911cdf8e383ee3b25184f8",
|
| 8 |
+
"max_tokens": 512
|
| 9 |
+
},
|
| 10 |
+
"data": {
|
| 11 |
+
"path": "data/generated/pilot-v2.jsonl",
|
| 12 |
+
"manifest": "data/generated/pilot-v2.manifest.json"
|
| 13 |
+
},
|
| 14 |
+
"seed": 7,
|
| 15 |
+
"epochs": 6,
|
| 16 |
+
"batch_size": 32,
|
| 17 |
+
"learning_rate": 0.001,
|
| 18 |
+
"device": "auto",
|
| 19 |
+
"artifact_dir": "artifacts/pilot-v2-direct",
|
| 20 |
+
"target_bundle_bytes": 67108864
|
| 21 |
+
}
|
configs/public-release.json
CHANGED
|
@@ -15,6 +15,8 @@
|
|
| 15 |
"README.md",
|
| 16 |
"configs/pilot-diffusion.json",
|
| 17 |
"configs/pilot.json",
|
|
|
|
|
|
|
| 18 |
"configs/public-release.json",
|
| 19 |
"docs/COMMUNITY.md",
|
| 20 |
"docs/HUB_PUBLISHING.md",
|
|
@@ -25,11 +27,18 @@
|
|
| 25 |
"docs/PUBLIC_RELEASE.md",
|
| 26 |
"docs/assets/inlevel9-signature-dark.png",
|
| 27 |
"docs/assets/inlevel9-signature.png",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
"docs/publications/ARXIV.md",
|
| 29 |
"docs/publications/LICENSE.md",
|
| 30 |
"docs/publications/README.md",
|
| 31 |
"docs/publications/vons-paper-v1.0.pdf",
|
|
|
|
| 32 |
"docs/publications/vons-tech-report-v1.0.1.pdf",
|
|
|
|
| 33 |
"pyproject.toml",
|
| 34 |
"sdk/typescript/LICENSE",
|
| 35 |
"sdk/typescript/README.md",
|
|
@@ -39,10 +48,25 @@
|
|
| 39 |
"sdk/typescript/demo/main.ts",
|
| 40 |
"sdk/typescript/demo/parity.html",
|
| 41 |
"sdk/typescript/demo/parity.ts",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
"sdk/typescript/package-lock.json",
|
| 43 |
"sdk/typescript/package.json",
|
| 44 |
"sdk/typescript/src/index.ts",
|
| 45 |
"sdk/typescript/src/onnx.ts",
|
|
|
|
| 46 |
"sdk/typescript/test/index.test.mjs",
|
| 47 |
"sdk/typescript/test/onnx.test.mjs",
|
| 48 |
"sdk/typescript/test/parity.test.mjs",
|
|
@@ -53,14 +77,18 @@
|
|
| 53 |
"tests/test_calibration.py",
|
| 54 |
"tests/test_contract.py",
|
| 55 |
"tests/test_data.py",
|
|
|
|
| 56 |
"tests/test_evaluation.py",
|
|
|
|
| 57 |
"tests/test_merge_browser_benchmarks.py",
|
| 58 |
"tests/test_mind2web.py",
|
|
|
|
| 59 |
"tests/test_mind2web_import.py",
|
| 60 |
"tests/test_mind2web_manifest.py",
|
| 61 |
"tests/test_mind2web_prediction.py",
|
| 62 |
"tests/test_mind2web_replay.py",
|
| 63 |
"tests/test_mind2web_runner.py",
|
|
|
|
| 64 |
"tests/test_public_release.py",
|
| 65 |
"tests/test_quality_summary.py",
|
| 66 |
"tests/test_regression.py",
|
|
@@ -76,6 +104,7 @@
|
|
| 76 |
"tools/export_shared_bundle.py",
|
| 77 |
"tools/freeze_publication_packet.py",
|
| 78 |
"tools/import_mind2web.py",
|
|
|
|
| 79 |
"tools/merge_browser_benchmarks.py",
|
| 80 |
"tools/predict_mind2web.py",
|
| 81 |
"tools/prepare_public_release.py",
|
|
@@ -93,8 +122,10 @@
|
|
| 93 |
"vons/contract.py",
|
| 94 |
"vons/data.py",
|
| 95 |
"vons/evaluation.py",
|
|
|
|
| 96 |
"vons/kasi.py",
|
| 97 |
"vons/mind2web.py",
|
|
|
|
| 98 |
"vons/models.py",
|
| 99 |
"vons/ollama.py",
|
| 100 |
"vons/research.py",
|
|
@@ -103,8 +134,15 @@
|
|
| 103 |
"reviewed_binaries": {
|
| 104 |
"docs/assets/inlevel9-signature.png": "f7514b56b2a93293054b3d22634ac5f0cb11e00a370f53d8d9f934c55a21bbb5",
|
| 105 |
"docs/assets/inlevel9-signature-dark.png": "d6645528158842af77a89de97361fa0c521fbc5a3c0ea6e996f9c912cfcfe9b5",
|
|
|
|
| 106 |
"docs/publications/vons-paper-v1.0.pdf": "ae0e09c60e17a98b52bb25c611b2489d10b18bd98d87974f2e8dd25d746a5fc8",
|
| 107 |
-
"docs/publications/vons-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
},
|
| 109 |
"software_license": "LicenseRef-Vons-Community-Commercial-1.0",
|
| 110 |
"article_license": "CC-BY-4.0"
|
|
|
|
| 15 |
"README.md",
|
| 16 |
"configs/pilot-diffusion.json",
|
| 17 |
"configs/pilot.json",
|
| 18 |
+
"configs/pilot-v2-diffusion.json",
|
| 19 |
+
"configs/pilot-v2-direct.json",
|
| 20 |
"configs/public-release.json",
|
| 21 |
"docs/COMMUNITY.md",
|
| 22 |
"docs/HUB_PUBLISHING.md",
|
|
|
|
| 27 |
"docs/PUBLIC_RELEASE.md",
|
| 28 |
"docs/assets/inlevel9-signature-dark.png",
|
| 29 |
"docs/assets/inlevel9-signature.png",
|
| 30 |
+
"docs/assets/vons-logo.png",
|
| 31 |
+
"docs/assets/vons-extension-icon-128.png",
|
| 32 |
+
"docs/assets/vons-extension-icon-16.png",
|
| 33 |
+
"docs/assets/vons-extension-icon-32.png",
|
| 34 |
+
"docs/assets/vons-extension-icon-48.png",
|
| 35 |
"docs/publications/ARXIV.md",
|
| 36 |
"docs/publications/LICENSE.md",
|
| 37 |
"docs/publications/README.md",
|
| 38 |
"docs/publications/vons-paper-v1.0.pdf",
|
| 39 |
+
"docs/publications/vons-paper-v1.1.pdf",
|
| 40 |
"docs/publications/vons-tech-report-v1.0.1.pdf",
|
| 41 |
+
"docs/publications/vons-tech-report-v1.1.pdf",
|
| 42 |
"pyproject.toml",
|
| 43 |
"sdk/typescript/LICENSE",
|
| 44 |
"sdk/typescript/README.md",
|
|
|
|
| 48 |
"sdk/typescript/demo/main.ts",
|
| 49 |
"sdk/typescript/demo/parity.html",
|
| 50 |
"sdk/typescript/demo/parity.ts",
|
| 51 |
+
"sdk/typescript/extension/ONNX_RUNTIME_NOTICES.md",
|
| 52 |
+
"sdk/typescript/extension/PRIVACY.md",
|
| 53 |
+
"sdk/typescript/extension/README.md",
|
| 54 |
+
"sdk/typescript/extension/background.mjs",
|
| 55 |
+
"sdk/typescript/extension/build.mjs",
|
| 56 |
+
"sdk/typescript/extension/bundle.ts",
|
| 57 |
+
"sdk/typescript/extension/manifest.json",
|
| 58 |
+
"sdk/typescript/extension/panel.css",
|
| 59 |
+
"sdk/typescript/extension/panel.html",
|
| 60 |
+
"sdk/typescript/extension/panel.ts",
|
| 61 |
+
"sdk/typescript/extension/privacy.html",
|
| 62 |
+
"sdk/typescript/extension/storage.ts",
|
| 63 |
+
"sdk/typescript/extension/tsconfig.json",
|
| 64 |
+
"sdk/typescript/extension/worker.ts",
|
| 65 |
"sdk/typescript/package-lock.json",
|
| 66 |
"sdk/typescript/package.json",
|
| 67 |
"sdk/typescript/src/index.ts",
|
| 68 |
"sdk/typescript/src/onnx.ts",
|
| 69 |
+
"sdk/typescript/test/extension.test.mjs",
|
| 70 |
"sdk/typescript/test/index.test.mjs",
|
| 71 |
"sdk/typescript/test/onnx.test.mjs",
|
| 72 |
"sdk/typescript/test/parity.test.mjs",
|
|
|
|
| 77 |
"tests/test_calibration.py",
|
| 78 |
"tests/test_contract.py",
|
| 79 |
"tests/test_data.py",
|
| 80 |
+
"tests/test_diffusion_model.py",
|
| 81 |
"tests/test_evaluation.py",
|
| 82 |
+
"tests/test_inference.py",
|
| 83 |
"tests/test_merge_browser_benchmarks.py",
|
| 84 |
"tests/test_mind2web.py",
|
| 85 |
+
"tests/test_mind2web_adapter.py",
|
| 86 |
"tests/test_mind2web_import.py",
|
| 87 |
"tests/test_mind2web_manifest.py",
|
| 88 |
"tests/test_mind2web_prediction.py",
|
| 89 |
"tests/test_mind2web_replay.py",
|
| 90 |
"tests/test_mind2web_runner.py",
|
| 91 |
+
"tests/test_onnx_export.py",
|
| 92 |
"tests/test_public_release.py",
|
| 93 |
"tests/test_quality_summary.py",
|
| 94 |
"tests/test_regression.py",
|
|
|
|
| 104 |
"tools/export_shared_bundle.py",
|
| 105 |
"tools/freeze_publication_packet.py",
|
| 106 |
"tools/import_mind2web.py",
|
| 107 |
+
"tools/index_mind2web_predictions.py",
|
| 108 |
"tools/merge_browser_benchmarks.py",
|
| 109 |
"tools/predict_mind2web.py",
|
| 110 |
"tools/prepare_public_release.py",
|
|
|
|
| 122 |
"vons/contract.py",
|
| 123 |
"vons/data.py",
|
| 124 |
"vons/evaluation.py",
|
| 125 |
+
"vons/inference.py",
|
| 126 |
"vons/kasi.py",
|
| 127 |
"vons/mind2web.py",
|
| 128 |
+
"vons/mind2web_adapter.py",
|
| 129 |
"vons/models.py",
|
| 130 |
"vons/ollama.py",
|
| 131 |
"vons/research.py",
|
|
|
|
| 134 |
"reviewed_binaries": {
|
| 135 |
"docs/assets/inlevel9-signature.png": "f7514b56b2a93293054b3d22634ac5f0cb11e00a370f53d8d9f934c55a21bbb5",
|
| 136 |
"docs/assets/inlevel9-signature-dark.png": "d6645528158842af77a89de97361fa0c521fbc5a3c0ea6e996f9c912cfcfe9b5",
|
| 137 |
+
"docs/assets/vons-logo.png": "1f557be52834662b480d16048b82ad8b03805af6aa1466f1b2d5add85ed74a5e",
|
| 138 |
"docs/publications/vons-paper-v1.0.pdf": "ae0e09c60e17a98b52bb25c611b2489d10b18bd98d87974f2e8dd25d746a5fc8",
|
| 139 |
+
"docs/publications/vons-paper-v1.1.pdf": "8e9ff7d768979411db2333f37111692b5b8426bdaec21d35954c7038d047f321",
|
| 140 |
+
"docs/publications/vons-tech-report-v1.0.1.pdf": "33076d9e52d4a073060066215dd390e3b0bf0cdef48c5ffb2a7c5189854254ed",
|
| 141 |
+
"docs/publications/vons-tech-report-v1.1.pdf": "9e6aff46a271324f82db594155ea6b3a7c51c5cfe9b192e52f2b38d92ca84af3",
|
| 142 |
+
"docs/assets/vons-extension-icon-16.png": "1a8d94bade23fed4e84c3ecca64f2904a575f1543981250f0896a17ecc88ffee",
|
| 143 |
+
"docs/assets/vons-extension-icon-32.png": "cc32b367c60bdcfd0b07a8b1b8771cfc064ffd45a3a35dfc764136bc05cb97c4",
|
| 144 |
+
"docs/assets/vons-extension-icon-48.png": "7ca6d66a7548ff8a8c42bf6bc4e0b123e2bca8abf3ba8366bab6690d9b44f101",
|
| 145 |
+
"docs/assets/vons-extension-icon-128.png": "6195919e1368d1695c3346097b216b38f170099080d53c014cc2a4162a7f25d6"
|
| 146 |
},
|
| 147 |
"software_license": "LicenseRef-Vons-Community-Commercial-1.0",
|
| 148 |
"article_license": "CC-BY-4.0"
|
docs/COMMUNITY.md
CHANGED
|
@@ -10,7 +10,7 @@ report what works or fails.
|
|
| 10 |
| Your goal | Entry point | Current availability |
|
| 11 |
| --- | --- | --- |
|
| 12 |
| Understand Vons | [README](../README.md) and [model card](MODEL_CARD.md) | Included in this source preview |
|
| 13 |
-
| Read the research | [Paper and Tech Report](publications/README.md) |
|
| 14 |
| Try without weights | The walkthrough below | Deterministic contract and synthetic data only |
|
| 15 |
| Run model inference | [TypeScript SDK](../sdk/typescript/README.md) | Requires separately obtained, reviewed model/tokenizer assets |
|
| 16 |
| Share experience | [Experience report](https://github.com/inlevel9-com/Vons/issues/new?template=experience.yml) | GitHub account required to submit |
|
|
@@ -20,8 +20,9 @@ report what works or fails.
|
|
| 20 |
The destinations are [GitHub: inlevel9-com/Vons](https://github.com/inlevel9-com/Vons)
|
| 21 |
and [Hugging Face: INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons).
|
| 22 |
The source preview includes documentation and GitHub issue forms for feedback.
|
| 23 |
-
The paper and Tech Report
|
| 24 |
-
|
|
|
|
| 25 |
The current software terms require a separate agreement for
|
| 26 |
enterprise/commercial use, including enterprise evaluation. Reading the paper
|
| 27 |
under its article terms is separate from software use.
|
|
|
|
| 10 |
| Your goal | Entry point | Current availability |
|
| 11 |
| --- | --- | --- |
|
| 12 |
| Understand Vons | [README](../README.md) and [model card](MODEL_CARD.md) | Included in this source preview |
|
| 13 |
+
| Read the research | [Paper and Tech Report](publications/README.md) | Reviewed PDFs and exact verification scope are included |
|
| 14 |
| Try without weights | The walkthrough below | Deterministic contract and synthetic data only |
|
| 15 |
| Run model inference | [TypeScript SDK](../sdk/typescript/README.md) | Requires separately obtained, reviewed model/tokenizer assets |
|
| 16 |
| Share experience | [Experience report](https://github.com/inlevel9-com/Vons/issues/new?template=experience.yml) | GitHub account required to submit |
|
|
|
|
| 20 |
The destinations are [GitHub: inlevel9-com/Vons](https://github.com/inlevel9-com/Vons)
|
| 21 |
and [Hugging Face: INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons).
|
| 22 |
The source preview includes documentation and GitHub issue forms for feedback.
|
| 23 |
+
The paper and Tech Report are included as versioned PDFs; their index preserves
|
| 24 |
+
exact hashes, evidence boundaries and arXiv status. A source upload does not
|
| 25 |
+
provide hosted model inference.
|
| 26 |
The current software terms require a separate agreement for
|
| 27 |
enterprise/commercial use, including enterprise evaluation. Reading the paper
|
| 28 |
under its article terms is separate from software use.
|
docs/HUB_PUBLISHING.md
CHANGED
|
@@ -1,9 +1,10 @@
|
|
| 1 |
# Upload the Vons source preview to Hugging Face
|
| 2 |
|
| 3 |
-
This guide covers the completed upload and
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
|
|
|
| 7 |
|
| 8 |
## Destination and scope
|
| 9 |
|
|
@@ -20,6 +21,13 @@ public model card now uses the Vons Community and Commercial License 1.0.
|
|
| 20 |
The article CC BY 4.0 terms are separate. A later documentation update records
|
| 21 |
the arXiv submission status without adding models or draft material.
|
| 22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
A commit to this public repository makes uploaded files publicly readable.
|
| 24 |
Uploading source does not create hosted inference, pretrained weights or a
|
| 25 |
working `from_pretrained` package. The card states those limits explicitly.
|
|
@@ -57,10 +65,11 @@ Run the source audit from the Vons checkout:
|
|
| 57 |
python3 tools/prepare_public_release.py
|
| 58 |
```
|
| 59 |
|
| 60 |
-
The initial uploaded snapshot is `releases/huggingface-source-v8`.
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
|
|
|
| 64 |
|
| 65 |
This export is a snapshot. If you edit any public source or documentation after
|
| 66 |
preparation, create a fresh versioned directory and use that path below:
|
|
@@ -105,19 +114,19 @@ Open [the model card](https://huggingface.co/INLEVEL9/Vons) and
|
|
| 105 |
It must no longer identify the software as Apache-2.0 or CC BY 4.0.
|
| 106 |
- Root `LICENSE` and `sdk/typescript/LICENSE` contain the same custom terms.
|
| 107 |
- Source paths such as `vons/`, `tools/`, `tests/` and `sdk/typescript/` are present.
|
| 108 |
-
- The Paper and Tech Report links open the
|
| 109 |
Their digests match the publication index. No manuscript source bundle or
|
| 110 |
private review material is uploaded.
|
| 111 |
-
-
|
| 112 |
|
| 113 |
Save the resulting Hub commit URL with the local release manifest. GitHub
|
| 114 |
publication, manuscript publication and a hosted demo are separate actions.
|
| 115 |
For subsequent source changes, create a new reviewed export and repeat the same
|
| 116 |
upload command with its new path. Add research files only after their separate
|
| 117 |
-
|
| 118 |
|
| 119 |
The initial Hub upload and publication-document update were owner-authorized.
|
| 120 |
-
GitHub commits, pushes and visibility changes remain owner-operated.
|
| 121 |
-
|
| 122 |
-
|
| 123 |
Official CLI/upload guidance checked 2026-09-25.
|
|
|
|
| 1 |
# Upload the Vons source preview to Hugging Face
|
| 2 |
|
| 3 |
+
This guide covers the completed initial upload and subsequent reviewed updates.
|
| 4 |
+
The current v1.1 snapshot contains 128 reviewed files, including the source,
|
| 5 |
+
extension source, Vons logo, current Paper and Tech Report PDFs and their
|
| 6 |
+
historical reviewed versions. Drafts, source manuscript bundles, model weights,
|
| 7 |
+
restricted data and private records are excluded.
|
| 8 |
|
| 9 |
## Destination and scope
|
| 10 |
|
|
|
|
| 21 |
The article CC BY 4.0 terms are separate. A later documentation update records
|
| 22 |
the arXiv submission status without adding models or draft material.
|
| 23 |
|
| 24 |
+
The reviewed v1.1 update uses the local snapshot
|
| 25 |
+
`releases/huggingface-source-v13`. It adds the 21-page Preprint v1.1, the 29-page
|
| 26 |
+
Technical Report v1.1, post-freeze implementation source and tests, Chrome
|
| 27 |
+
extension source and the owner-supplied Vons logo. It does not add manuscript
|
| 28 |
+
source bundles, claim ledgers, model weights, tokenizer assets or benchmark
|
| 29 |
+
data.
|
| 30 |
+
|
| 31 |
A commit to this public repository makes uploaded files publicly readable.
|
| 32 |
Uploading source does not create hosted inference, pretrained weights or a
|
| 33 |
working `from_pretrained` package. The card states those limits explicitly.
|
|
|
|
| 65 |
python3 tools/prepare_public_release.py
|
| 66 |
```
|
| 67 |
|
| 68 |
+
The initial uploaded snapshot is `releases/huggingface-source-v8`. The current
|
| 69 |
+
v1.1 snapshot is `releases/huggingface-source-v13`; it contains 128 reviewed
|
| 70 |
+
files plus a local `RELEASE_MANIFEST.json` receipt with every file's byte count
|
| 71 |
+
and SHA-256 digest. Its root README is the Hub model card; the working-tree
|
| 72 |
+
README remains the GitHub landing page.
|
| 73 |
|
| 74 |
This export is a snapshot. If you edit any public source or documentation after
|
| 75 |
preparation, create a fresh versioned directory and use that path below:
|
|
|
|
| 114 |
It must no longer identify the software as Apache-2.0 or CC BY 4.0.
|
| 115 |
- Root `LICENSE` and `sdk/typescript/LICENSE` contain the same custom terms.
|
| 116 |
- Source paths such as `vons/`, `tools/`, `tests/` and `sdk/typescript/` are present.
|
| 117 |
+
- The Paper and Tech Report links open the 21-page v1.1 paper and 29-page v1.1 report.
|
| 118 |
Their digests match the publication index. No manuscript source bundle or
|
| 119 |
private review material is uploaded.
|
| 120 |
+
- The Vons and INLEVEL9 logo links resolve, and a signed-out visitor can read the card.
|
| 121 |
|
| 122 |
Save the resulting Hub commit URL with the local release manifest. GitHub
|
| 123 |
publication, manuscript publication and a hosted demo are separate actions.
|
| 124 |
For subsequent source changes, create a new reviewed export and repeat the same
|
| 125 |
upload command with its new path. Add research files only after their separate
|
| 126 |
+
review and an explicit public-allowlist update.
|
| 127 |
|
| 128 |
The initial Hub upload and publication-document update were owner-authorized.
|
| 129 |
+
GitHub commits, pushes and visibility changes remain owner-operated. The v1.1
|
| 130 |
+
Hub source update includes reviewed extension source, but generated extension
|
| 131 |
+
packages and imported local models remain excluded.
|
| 132 |
Official CLI/upload guidance checked 2026-09-25.
|
docs/HUGGING_FACE.md
CHANGED
|
@@ -11,6 +11,8 @@ tags:
|
|
| 11 |
- onnx
|
| 12 |
---
|
| 13 |
|
|
|
|
|
|
|
| 14 |

|
| 15 |
|
| 16 |
# Vons
|
|
@@ -57,17 +59,23 @@ an organization card, source preview and later a separately validated Space.
|
|
| 57 |
|
| 58 |
| Publication | Download |
|
| 59 |
| --- | --- |
|
| 60 |
-
| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows —
|
| 61 |
-
| Vons: Compact Decision Models for Frontier-Agent Workflows — Technical Report v1.
|
| 62 |
|
| 63 |
Articles and original figures use **CC BY 4.0**. Their version, SHA-256 digests
|
| 64 |
and verification scope are in the [publication index](../docs/publications/README.md).
|
| 65 |
-
These preprints preserve negative results
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
|
|
|
| 69 |
acceptance is claimed.
|
| 70 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
## Local use
|
| 72 |
|
| 73 |
```sh
|
|
|
|
| 11 |
- onnx
|
| 12 |
---
|
| 13 |
|
| 14 |
+

|
| 15 |
+
|
| 16 |

|
| 17 |
|
| 18 |
# Vons
|
|
|
|
| 59 |
|
| 60 |
| Publication | Download |
|
| 61 |
| --- | --- |
|
| 62 |
+
| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows — Preprint v1.1 | [PDF · 21 pages](../docs/publications/vons-paper-v1.1.pdf) |
|
| 63 |
+
| Vons: Compact Decision Models for Frontier-Agent Workflows — Technical Report v1.1 | [PDF · 29 pages](../docs/publications/vons-tech-report-v1.1.pdf) |
|
| 64 |
|
| 65 |
Articles and original figures use **CC BY 4.0**. Their version, SHA-256 digests
|
| 66 |
and verification scope are in the [publication index](../docs/publications/README.md).
|
| 67 |
+
These preprints preserve negative results, missing measurements and the boundary
|
| 68 |
+
between frozen v1 evidence and the separately labelled post-freeze v2 handoff.
|
| 69 |
+
The paper was submitted to arXiv under **submission number 8127259**; the last
|
| 70 |
+
checked status on **2026-09-25** is **on hold** (undergoing checks). This tracking
|
| 71 |
+
number is not a public arXiv article identifier. No announcement or peer-review
|
| 72 |
acceptance is claimed.
|
| 73 |
|
| 74 |
+
Chrome Web Store version **0.1.0** passed review and is publicly available as
|
| 75 |
+
[Vons — Local Decisions](https://chromewebstore.google.com/detail/vons-%E2%80%94-local-decisions/cbmjkokoojinfgafahncmjlfmicinide),
|
| 76 |
+
verified on 2026-09-25. Store availability is not evidence of model quality,
|
| 77 |
+
scientific validity or production readiness.
|
| 78 |
+
|
| 79 |
## Local use
|
| 80 |
|
| 81 |
```sh
|
docs/MODEL_CARD.md
CHANGED
|
@@ -22,15 +22,21 @@ Runtime libraries, complete download size and memory use are distinct quantities
|
|
| 22 |
## Evaluation and limitations
|
| 23 |
|
| 24 |
Historical local experiments include synthetic pilots and an evaluation-only
|
| 25 |
-
external benchmark diagnostic.
|
| 26 |
-
|
| 27 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
[paper and Tech Report](publications/README.md), with exact evidence references.
|
| 29 |
Unreviewed historical tables are not distributed as current results.
|
| 30 |
|
| 31 |
The ONNX candidate head does not support general score questions. Python and
|
| 32 |
-
TypeScript
|
| 33 |
-
|
|
|
|
|
|
|
| 34 |
|
| 35 |
Do not use model confidence as a permission grant or assume it is calibrated.
|
| 36 |
Vons cannot execute tools or approve high-risk actions.
|
|
|
|
| 22 |
## Evaluation and limitations
|
| 23 |
|
| 24 |
Historical local experiments include synthetic pilots and an evaluation-only
|
| 25 |
+
external benchmark diagnostic. A separately labelled post-freeze v2 handoff
|
| 26 |
+
records dynamic per-request padding, balanced four-class synthetic runs,
|
| 27 |
+
vector-scaling calibration, a token-conditioned Diffusion head and label-free
|
| 28 |
+
Mind2Web context compression. Those bounded synthetic, local-CPU and
|
| 29 |
+
conditional-selection results do not establish general task quality, production
|
| 30 |
+
readiness, calibrated safety or a Direct/Diffusion superiority claim. Reviewed
|
| 31 |
+
numerical claims belong to the versioned
|
| 32 |
[paper and Tech Report](publications/README.md), with exact evidence references.
|
| 33 |
Unreviewed historical tables are not distributed as current results.
|
| 34 |
|
| 35 |
The ONNX candidate head does not support general score questions. Python and
|
| 36 |
+
TypeScript now both pad to the longest live tokenized candidate within the
|
| 37 |
+
manifest budget, but their broader public contract semantics are not claimed to
|
| 38 |
+
be identical. Browser smoke, local CPU timing, graph-partition parity and memory
|
| 39 |
+
evidence remain separate checks.
|
| 40 |
|
| 41 |
Do not use model confidence as a permission grant or assume it is calibrated.
|
| 42 |
Vons cannot execute tools or approve high-risk actions.
|
docs/PUBLIC_RELEASE.md
CHANGED
|
@@ -8,7 +8,7 @@ tool independently rejects any Git candidate outside that exact file list.
|
|
| 8 |
|
| 9 |
Included: Python and TypeScript source, tests, source demo files, the SDK
|
| 10 |
dependency lockfile, pilot configuration, reviewed public documentation and the
|
| 11 |
-
owner-supplied INLEVEL9
|
| 12 |
`docs/publications/README.md`.
|
| 13 |
|
| 14 |
Excluded: all local datasets (including synthetic outputs), model weights,
|
|
@@ -32,8 +32,8 @@ Each export is a new directory containing only allowlisted files and a
|
|
| 32 |
`RELEASE_MANIFEST.json` with byte counts and SHA-256 hashes. Existing output
|
| 33 |
directories are never overwritten. Use a new versioned output path after any
|
| 34 |
source change. The Hugging Face export substitutes its model card as the root
|
| 35 |
-
README. Both exports contain source
|
| 36 |
-
no pretrained assets.
|
| 37 |
|
| 38 |
The audit detects known token/key patterns, personal home paths, private writing
|
| 39 |
session URLs, unexpected Git files, symlinks and changed reviewed binaries.
|
|
@@ -49,8 +49,8 @@ remove already committed material from history. If existing remote history is
|
|
| 49 |
later attached, review that history independently before making it public.
|
| 50 |
|
| 51 |
The CI workflow checks the release allowlist, Python tests and lint, and the SDK
|
| 52 |
-
tests
|
| 53 |
-
tests, configuration or the workflow changes; documentation-only edits do not
|
| 54 |
trigger the full suite. A manual run remains available. SDK tokenizer tests use
|
| 55 |
synthetic fixtures and require no private pilot assets. It has read-only repository permissions and
|
| 56 |
does not publish packages or artifacts. Checks that require unavailable optional
|
|
@@ -62,11 +62,21 @@ those files; do not publish the historical inputs merely to remove that skip.
|
|
| 62 |
## Hugging Face
|
| 63 |
|
| 64 |
Upload only a reviewed versioned Hugging Face export to `INLEVEL9/Vons`.
|
| 65 |
-
The
|
|
|
|
| 66 |
[step-by-step upload guide](HUB_PUBLISHING.md). Do not select the development workspace in the
|
| 67 |
Hub upload dialog. Use explicit allow/ignore patterns if using `upload_folder`;
|
| 68 |
its handling of ignore files depends on the client version and upload method.
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
This source preview has no weights and no inference widget. A later weight
|
| 71 |
release needs its own exact file inventory, upstream notices, model/data
|
| 72 |
provenance and runtime validation. Do not turn the ignored checkpoint directory
|
|
|
|
| 8 |
|
| 9 |
Included: Python and TypeScript source, tests, source demo files, the SDK
|
| 10 |
dependency lockfile, pilot configuration, reviewed public documentation and the
|
| 11 |
+
owner-supplied Vons and INLEVEL9 logos. Reviewed publications are listed separately in
|
| 12 |
`docs/publications/README.md`.
|
| 13 |
|
| 14 |
Excluded: all local datasets (including synthetic outputs), model weights,
|
|
|
|
| 32 |
`RELEASE_MANIFEST.json` with byte counts and SHA-256 hashes. Existing output
|
| 33 |
directories are never overwritten. Use a new versioned output path after any
|
| 34 |
source change. The Hugging Face export substitutes its model card as the root
|
| 35 |
+
README. Both exports contain source, the current v1.1 Paper and Tech Report and
|
| 36 |
+
their two reviewed historical PDFs, with no pretrained assets.
|
| 37 |
|
| 38 |
The audit detects known token/key patterns, personal home paths, private writing
|
| 39 |
session URLs, unexpected Git files, symlinks and changed reviewed binaries.
|
|
|
|
| 49 |
later attached, review that history independently before making it public.
|
| 50 |
|
| 51 |
The CI workflow checks the release allowlist, Python tests and lint, and the SDK
|
| 52 |
+
tests, three browser builds, and extension type checking/building. Push/pull-request checks run when source,
|
| 53 |
+
tests, configuration, packaged extension images or the workflow changes; documentation-only edits do not
|
| 54 |
trigger the full suite. A manual run remains available. SDK tokenizer tests use
|
| 55 |
synthetic fixtures and require no private pilot assets. It has read-only repository permissions and
|
| 56 |
does not publish packages or artifacts. Checks that require unavailable optional
|
|
|
|
| 62 |
## Hugging Face
|
| 63 |
|
| 64 |
Upload only a reviewed versioned Hugging Face export to `INLEVEL9/Vons`.
|
| 65 |
+
The initial verified upload used `releases/huggingface-source-v8`. The reviewed
|
| 66 |
+
v1.1 update is prepared as `releases/huggingface-source-v13`; see the
|
| 67 |
[step-by-step upload guide](HUB_PUBLISHING.md). Do not select the development workspace in the
|
| 68 |
Hub upload dialog. Use explicit allow/ignore patterns if using `upload_folder`;
|
| 69 |
its handling of ignore files depends on the client version and upload method.
|
| 70 |
|
| 71 |
+
The current checkout also includes the [Chrome extension source](../sdk/typescript/extension/README.md),
|
| 72 |
+
which was not part of that initial 97-file upload but is included in the v1.1
|
| 73 |
+
source update. Generated extension packages remain local and
|
| 74 |
+
ignored; distribute only their reviewed contents, with dependency notices and
|
| 75 |
+
without imported models or prompts. Chrome Web Store version 0.1.0 was submitted
|
| 76 |
+
on 2026-09-25, passed review and had its public listing independently read back
|
| 77 |
+
the same day: [Vons — Local Decisions](https://chromewebstore.google.com/detail/vons-%E2%80%94-local-decisions/cbmjkokoojinfgafahncmjlfmicinide).
|
| 78 |
+
That listing does not establish model quality or production readiness.
|
| 79 |
+
|
| 80 |
This source preview has no weights and no inference widget. A later weight
|
| 81 |
release needs its own exact file inventory, upstream notices, model/data
|
| 82 |
provenance and runtime validation. Do not turn the ignored checkpoint directory
|
docs/assets/vons-extension-icon-128.png
ADDED
|
|
docs/assets/vons-extension-icon-16.png
ADDED
|
|
docs/assets/vons-extension-icon-32.png
ADDED
|
|
docs/assets/vons-extension-icon-48.png
ADDED
|
|
docs/assets/vons-logo.png
ADDED
|
docs/publications/README.md
CHANGED
|
@@ -1,7 +1,9 @@
|
|
| 1 |
# Paper and Tech Report
|
| 2 |
|
| 3 |
-
The reviewed PDFs are included in this release.
|
| 4 |
-
|
|
|
|
|
|
|
| 5 |
|
| 6 |
Author: **Kwangseob Ahn**
|
| 7 |
Affiliation: **INLEVEL9 / SEJONG UNIV.**
|
|
@@ -9,23 +11,37 @@ Contact: **oswarld@inlevel9.com**
|
|
| 9 |
|
| 10 |
| Publication | Version | Download |
|
| 11 |
| --- | --- | --- |
|
| 12 |
-
| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows |
|
| 13 |
-
| Vons: Compact Decision Models for Frontier-Agent Workflows | Technical Report v1.
|
| 14 |
|
| 15 |
## Verification and provenance
|
| 16 |
|
| 17 |
-
The final documents were reviewed against the frozen evidence packet
|
| 18 |
-
references
|
| 19 |
-
|
| 20 |
-
values
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
| File | SHA-256 |
|
| 26 |
| --- | --- |
|
| 27 |
-
| `vons-paper-v1.
|
| 28 |
-
| `vons-tech-report-v1.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
Only these exact PDFs are permitted by the public allowlist. Drafts, prompts,
|
| 31 |
claim ledgers, review conversations, source bundles and restricted benchmark data
|
|
|
|
| 1 |
# Paper and Tech Report
|
| 2 |
|
| 3 |
+
The reviewed v1.1 PDFs are included in this release. They preserve the frozen
|
| 4 |
+
v1 evidence while reporting a separately labelled post-freeze remediation
|
| 5 |
+
handoff. Publication does not imply independent re-execution, external
|
| 6 |
+
generalization, production readiness or peer acceptance.
|
| 7 |
|
| 8 |
Author: **Kwangseob Ahn**
|
| 9 |
Affiliation: **INLEVEL9 / SEJONG UNIV.**
|
|
|
|
| 11 |
|
| 12 |
| Publication | Version | Download |
|
| 13 |
| --- | --- | --- |
|
| 14 |
+
| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows | Preprint v1.1 | [PDF · 21 pages](vons-paper-v1.1.pdf) |
|
| 15 |
+
| Vons: Compact Decision Models for Frontier-Agent Workflows | Technical Report v1.1 | [PDF · 29 pages](vons-tech-report-v1.1.pdf) |
|
| 16 |
|
| 17 |
## Verification and provenance
|
| 18 |
|
| 19 |
+
The final documents were reviewed against the frozen evidence packet, primary
|
| 20 |
+
references and the post-freeze handoff. Claude Science did not access or rerun
|
| 21 |
+
the post-freeze files; Codex independently checked the handed-off digests,
|
| 22 |
+
values, implementation tests and local repository state. Every page of both
|
| 23 |
+
PDFs was visually inspected. Independently checked local SHA-256 values match
|
| 24 |
+
the reviewed release versions. All pages render as US Letter, all fonts are
|
| 25 |
+
embedded, and no encryption, PDF JavaScript, launch action or embedded
|
| 26 |
+
attachment was found. These are genuine Typst-generated PDFs. Science rebuilt
|
| 27 |
+
both documents from their v1.1 release source; the private source bundles,
|
| 28 |
+
claim ledgers and verification records are intentionally excluded from the
|
| 29 |
+
public software release.
|
| 30 |
|
| 31 |
| File | SHA-256 |
|
| 32 |
| --- | --- |
|
| 33 |
+
| `vons-paper-v1.1.pdf` | `8e9ff7d768979411db2333f37111692b5b8426bdaec21d35954c7038d047f321` |
|
| 34 |
+
| `vons-tech-report-v1.1.pdf` | `9e6aff46a271324f82db594155ea6b3a7c51c5cfe9b192e52f2b38d92ca84af3` |
|
| 35 |
+
|
| 36 |
+
## Historical reviewed versions
|
| 37 |
+
|
| 38 |
+
The prior PDFs remain available for provenance. They do not include the
|
| 39 |
+
post-freeze v2 remediation section.
|
| 40 |
+
|
| 41 |
+
| File | Version | SHA-256 |
|
| 42 |
+
| --- | --- | --- |
|
| 43 |
+
| `vons-paper-v1.0.pdf` | Paper v1.0 | `ae0e09c60e17a98b52bb25c611b2489d10b18bd98d87974f2e8dd25d746a5fc8` |
|
| 44 |
+
| `vons-tech-report-v1.0.1.pdf` | Technical Report v1.0.1 | `33076d9e52d4a073060066215dd390e3b0bf0cdef48c5ffb2a7c5189854254ed` |
|
| 45 |
|
| 46 |
Only these exact PDFs are permitted by the public allowlist. Drafts, prompts,
|
| 47 |
claim ledgers, review conversations, source bundles and restricted benchmark data
|
docs/publications/vons-paper-v1.1.pdf
ADDED
|
Binary file (131 Bytes). View file
|
|
|
docs/publications/vons-tech-report-v1.1.pdf
ADDED
|
Binary file (131 Bytes). View file
|
|
|
sdk/typescript/README.md
CHANGED
|
@@ -36,6 +36,11 @@ values, not the repeated benchmark protocol.
|
|
| 36 |
|
| 37 |
## Source preview
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
This package is private in npm metadata until an npm release is requested.
|
| 40 |
Its `files` allowlist includes source, this guide and the software license. Models, tokenizers,
|
| 41 |
datasets, generated demo JavaScript and dependencies are intentionally excluded
|
|
|
|
| 36 |
|
| 37 |
## Source preview
|
| 38 |
|
| 39 |
+
For a UI that does not require serving the development workspace, build the
|
| 40 |
+
[Chrome side-panel extension](extension/README.md) with `npm run build:extension`.
|
| 41 |
+
It packages the runtime locally and accepts a user-selected, hash-verified model
|
| 42 |
+
folder. The guide explains installation and the separate model requirement.
|
| 43 |
+
|
| 44 |
This package is private in npm metadata until an npm release is requested.
|
| 45 |
Its `files` allowlist includes source, this guide and the software license. Models, tokenizers,
|
| 46 |
datasets, generated demo JavaScript and dependencies are intentionally excluded
|
sdk/typescript/extension/ONNX_RUNTIME_NOTICES.md
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
sdk/typescript/extension/README.md
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Vons Chrome Extension
|
| 2 |
+
|
| 3 |
+
A Manifest V3 side panel for local candidate comparison. It supports full-graph
|
| 4 |
+
Direct and Diffusion Vons ONNX bundles, verifies exact manifest/asset hashes,
|
| 5 |
+
and keeps execution with the user. No model weight is included in this package.
|
| 6 |
+
|
| 7 |
+
## Build and install locally
|
| 8 |
+
|
| 9 |
+
Requires Node 22+ and Chrome 114+:
|
| 10 |
+
|
| 11 |
+
```sh
|
| 12 |
+
cd sdk/typescript
|
| 13 |
+
npm ci
|
| 14 |
+
npm test
|
| 15 |
+
npm run build:extension
|
| 16 |
+
```
|
| 17 |
+
|
| 18 |
+
Open `chrome://extensions`, enable Developer mode for your development profile,
|
| 19 |
+
choose **Load unpacked**, and select `sdk/typescript/dist/chrome-extension` from
|
| 20 |
+
the repository root. Pin **Vons — Local Decisions** in the toolbar and click it
|
| 21 |
+
to open the side panel. A store listing and automatic updates are not provided
|
| 22 |
+
by this local build. The generated folder includes all runtime JS/WASM, custom
|
| 23 |
+
software terms and third-party notices; it excludes model weights and local data.
|
| 24 |
+
|
| 25 |
+
For distribution, ZIP the *contents* of that generated directory so `manifest.json`
|
| 26 |
+
is at the archive root. Review the resulting package before Chrome Web Store
|
| 27 |
+
submission. Do not ZIP the workspace or your imported model folder.
|
| 28 |
+
|
| 29 |
+
## Chrome Web Store status
|
| 30 |
+
|
| 31 |
+
**Vons — Local Decisions**, version **0.1.0**, item ID
|
| 32 |
+
`cbmjkokoojinfgafahncmjlfmicinide`, was submitted to the Chrome Web Store on
|
| 33 |
+
**2026-09-25** after the owner approved the data-use declarations and final
|
| 34 |
+
submission. It subsequently passed review and the public listing was verified
|
| 35 |
+
on **2026-09-25**:
|
| 36 |
+
[install Vons — Local Decisions](https://chromewebstore.google.com/detail/vons-%E2%80%94-local-decisions/cbmjkokoojinfgafahncmjlfmicinide).
|
| 37 |
+
The package, English listing, icon, two screenshots, privacy disclosures and
|
| 38 |
+
reviewer instructions are included. The local unpacked installation remains
|
| 39 |
+
available above.
|
| 40 |
+
|
| 41 |
+
The store package contains runtime code and notices only. A compatible model
|
| 42 |
+
must be supplied by the user; the public source repository does not distribute
|
| 43 |
+
pretrained weights. Noncommercial use is free under the included software terms;
|
| 44 |
+
enterprise/commercial use requires a separate license.
|
| 45 |
+
|
| 46 |
+
## First comparison
|
| 47 |
+
|
| 48 |
+
1. Click **Import model folder** and choose the directory containing exactly one
|
| 49 |
+
`bundle-manifest-v1.json`, its full ONNX graph, external-data files and tokenizer.
|
| 50 |
+
Use a manifest produced by the Vons bundle tools. Head-only/shared-v1 partition
|
| 51 |
+
manifests use a different runtime contract and are not supported here.
|
| 52 |
+
2. Optionally keep the model on this device. Nothing is uploaded. The original
|
| 53 |
+
manifest and only the required runtime assets are retained, not checkpoint or
|
| 54 |
+
report files that happen to share the folder. SHA-256 verifies consistency
|
| 55 |
+
with the imported manifest, not the identity or trustworthiness of its author.
|
| 56 |
+
3. Enter context, a question and 2–32 unique candidates, one per line. **Use example**
|
| 57 |
+
supplies an illustrative prompt; its outcome is not a quality test.
|
| 58 |
+
4. Click **Compare candidates**. Read the proposed choice or explicit abstention,
|
| 59 |
+
uncalibrated model scores and single-run timings. No page action is executed.
|
| 60 |
+
|
| 61 |
+
WASM uses one thread and works without a WebGPU device. WebGPU is an explicit
|
| 62 |
+
experimental choice; failure is shown rather than silently falling back.
|
| 63 |
+
**Cancel** terminates the inference worker, and a subsequent comparison creates
|
| 64 |
+
a fresh runtime. A two-minute timeout also stops the worker. The model's actual
|
| 65 |
+
tokenizer enforces its aggregate input budget; long input is rejected rather than
|
| 66 |
+
silently truncated. Models larger than 128 MiB of selected runtime assets are
|
| 67 |
+
rejected before initialization. The paper's 64 MiB design target refers to model
|
| 68 |
+
assets, not the total extension plus runtime download.
|
| 69 |
+
|
| 70 |
+
## Privacy and host boundary
|
| 71 |
+
|
| 72 |
+
Only `sidePanel` is requested. There are no host permissions, content scripts,
|
| 73 |
+
page extraction, tool execution, network model fetches, analytics or account
|
| 74 |
+
requirements. Prompts/results are in memory only. IndexedDB stores a model only
|
| 75 |
+
when requested; **Remove model** deletes that retained copy. No Chrome Sync is
|
| 76 |
+
used. The bundled [privacy notice](privacy.html) describes actual handling.
|
| 77 |
+
The CSP restricts code and fetches to packaged extension resources and permits
|
| 78 |
+
WebAssembly compilation without permitting general JavaScript eval.
|
| 79 |
+
|
| 80 |
+
## Verification scope
|
| 81 |
+
|
| 82 |
+
Source tests cover exact file integrity, missing assets, path escapes, candidate
|
| 83 |
+
validation and unexpected extension permissions. On 2026-09-25, all 16 SDK tests,
|
| 84 |
+
strict extension type checking and the extension build passed. Real Chrome
|
| 85 |
+
testing of the generated interface over localhost verified Direct/WASM
|
| 86 |
+
inference, explicit token-budget rejection, cancellation followed by reuse,
|
| 87 |
+
optional model persistence and removal. The illustrative decision abstained;
|
| 88 |
+
this was a functionality check, not a model-quality benchmark.
|
| 89 |
+
|
| 90 |
+
The owner installed the unpacked extension; a subsequent independent check of
|
| 91 |
+
that actual side panel completed both Direct and Diffusion WASM inference using
|
| 92 |
+
the private pilot bundles. Both examples abstained. Direct took 326 ms for
|
| 93 |
+
inference and 212 ms for load/check; Diffusion took 3,238 ms and 159 ms. These are
|
| 94 |
+
single-run observations, not a latency benchmark or accuracy evidence. The
|
| 95 |
+
Diffusion ranking placed an unsuitable option first, reinforcing the need to
|
| 96 |
+
preserve abstention and human review. The imported models remain unpublished.
|
| 97 |
+
|
| 98 |
+
The installed-panel check exercises the packaged worker/WASM path under its
|
| 99 |
+
extension CSP. WebGPU, broader provider parity, external-task quality and
|
| 100 |
+
production readiness are not established by these checks. Store review
|
| 101 |
+
establishes listing-policy approval only, not scientific validity or safety.
|
| 102 |
+
Preserve the model manifest hash with each result.
|
| 103 |
+
|
| 104 |
+
Official references: [Side Panel API](https://developer.chrome.com/docs/extensions/reference/api/sidePanel),
|
| 105 |
+
[MV3 CSP](https://developer.chrome.com/docs/extensions/reference/manifest/content-security-policy),
|
| 106 |
+
[local development installation](https://developer.chrome.com/docs/extensions/get-started/tutorial/hello-world#load-unpacked).
|
sdk/typescript/extension/background.mjs
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// The side panel owns inference; this worker never reads or modifies a page.
|
| 2 |
+
const configurePanel = () => chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true });
|
| 3 |
+
chrome.runtime.onInstalled.addListener(configurePanel);
|
| 4 |
+
chrome.runtime.onStartup.addListener(configurePanel);
|
sdk/typescript/extension/build.mjs
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { build } from "esbuild";
|
| 2 |
+
import { readFile, writeFile, mkdir, copyFile, readdir, rm } from "node:fs/promises";
|
| 3 |
+
import { fileURLToPath } from "node:url";
|
| 4 |
+
import path from "node:path";
|
| 5 |
+
|
| 6 |
+
const root = fileURLToPath(new URL("../", import.meta.url));
|
| 7 |
+
const source = path.join(root, "extension");
|
| 8 |
+
const output = path.join(root, "dist/chrome-extension");
|
| 9 |
+
await rm(output, { recursive: true, force: true });
|
| 10 |
+
await mkdir(path.join(output, "runtime"), { recursive: true });
|
| 11 |
+
const result = await build({
|
| 12 |
+
absWorkingDir: root, entryPoints: ["extension/panel.ts", "extension/worker.ts"],
|
| 13 |
+
bundle: true, splitting: true, format: "esm", platform: "browser", target: "chrome114",
|
| 14 |
+
conditions: ["onnxruntime-web-use-extern-wasm"], outdir: output,
|
| 15 |
+
metafile: true, legalComments: "eof", minify: true,
|
| 16 |
+
});
|
| 17 |
+
for (const name of ["manifest.json", "background.mjs", "panel.html", "panel.css", "privacy.html"]) {
|
| 18 |
+
await copyFile(path.join(source, name), path.join(output, name));
|
| 19 |
+
}
|
| 20 |
+
await copyFile(path.join(root, "../../docs/assets/inlevel9-signature.png"), path.join(output, "brand.png"));
|
| 21 |
+
for (const size of [16, 32, 48, 128]) {
|
| 22 |
+
await copyFile(path.join(root, `../../docs/assets/vons-extension-icon-${size}.png`), path.join(output, `icon-${size}.png`));
|
| 23 |
+
}
|
| 24 |
+
await copyFile(path.join(root, "LICENSE"), path.join(output, "LICENSE"));
|
| 25 |
+
for (const name of ["ort-wasm-simd-threaded.mjs", "ort-wasm-simd-threaded.wasm",
|
| 26 |
+
"ort-wasm-simd-threaded.jsep.mjs", "ort-wasm-simd-threaded.jsep.wasm"]) {
|
| 27 |
+
await copyFile(path.join(root, "node_modules/onnxruntime-web/dist", name), path.join(output, "runtime", name));
|
| 28 |
+
}
|
| 29 |
+
const packages = new Set(["onnxruntime-web"]);
|
| 30 |
+
for (const input of Object.keys(result.metafile.inputs)) {
|
| 31 |
+
const parts = input.split("node_modules/").at(-1).split("/");
|
| 32 |
+
if (input.includes("node_modules/")) packages.add(parts[0].startsWith("@") ? `${parts[0]}/${parts[1]}` : parts[0]);
|
| 33 |
+
}
|
| 34 |
+
const notices = ["# Bundled third-party notices", "", "Vons source uses the accompanying LICENSE. Model weights are not included.", ""];
|
| 35 |
+
for (const name of [...packages].sort()) {
|
| 36 |
+
const packageRoot = path.join(root, "node_modules", name);
|
| 37 |
+
const metadata = JSON.parse(await readFile(path.join(packageRoot, "package.json"), "utf8"));
|
| 38 |
+
const files = (await readdir(packageRoot)).filter((file) => /^(?:licen[sc]e|copying|notice|thirdpartynotices)(?:\.|$)/i.test(file));
|
| 39 |
+
if (!files.length && name.startsWith("onnxruntime-")) {
|
| 40 |
+
if (metadata.version !== "1.30.0") throw new Error("Refresh the pinned ONNX Runtime notices before upgrading.");
|
| 41 |
+
notices.push(`## ${name} ${metadata.version}`, await readFile(path.join(source, "ONNX_RUNTIME_NOTICES.md"), "utf8"));
|
| 42 |
+
continue;
|
| 43 |
+
}
|
| 44 |
+
if (!files.length) throw new Error(`Missing third-party license notice: ${name}`);
|
| 45 |
+
notices.push(`## ${name} ${metadata.version}`, `Declared license: ${metadata.license}`, "");
|
| 46 |
+
for (const file of files) notices.push(`### ${file}`, "", await readFile(path.join(packageRoot, file), "utf8"), "");
|
| 47 |
+
}
|
| 48 |
+
await writeFile(path.join(output, "THIRD_PARTY_NOTICES.txt"), notices.join("\n"));
|
| 49 |
+
const manifest = JSON.parse(await readFile(path.join(output, "manifest.json"), "utf8"));
|
| 50 |
+
if (JSON.stringify(manifest.permissions) !== '["sidePanel"]' || manifest.host_permissions || manifest.content_scripts) throw new Error("Unexpected extension access.");
|
| 51 |
+
console.log(`Built ${output}; model weights excluded; ${packages.size} dependency notices included.`);
|
sdk/typescript/extension/bundle.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { validateRequest, type DecisionRequest } from "../src/index.ts";
|
| 2 |
+
import type { BundleManifest } from "../src/onnx.ts";
|
| 3 |
+
|
| 4 |
+
export interface LocalAsset { path: string; bytes: ArrayBuffer; }
|
| 5 |
+
export interface LocalBundle {
|
| 6 |
+
manifestPath: string;
|
| 7 |
+
manifestBytes: ArrayBuffer;
|
| 8 |
+
manifestHash: string;
|
| 9 |
+
backend: "direct" | "diffusion";
|
| 10 |
+
modelId: string;
|
| 11 |
+
assets: LocalAsset[];
|
| 12 |
+
}
|
| 13 |
+
export const MAX_BUNDLE_BYTES = 128 * 1024 * 1024;
|
| 14 |
+
export function safePath(path: string): boolean {
|
| 15 |
+
return path.length > 0 && !path.startsWith("/") && !path.includes("..")
|
| 16 |
+
&& !/[\\%?#:\u0000-\u001f]/.test(path)
|
| 17 |
+
&& path.split("/").every((part) => part.length > 0);
|
| 18 |
+
}
|
| 19 |
+
export async function digest(bytes: ArrayBuffer): Promise<string> {
|
| 20 |
+
const hash = await crypto.subtle.digest("SHA-256", bytes);
|
| 21 |
+
return Array.from(new Uint8Array(hash), (b) => b.toString(16).padStart(2, "0")).join("");
|
| 22 |
+
}
|
| 23 |
+
export function requiredFiles(bytes: ArrayBuffer): { manifest: BundleManifest; paths: string[] } {
|
| 24 |
+
if (bytes.byteLength > 2 * 1024 * 1024) throw new Error("Manifest exceeds 2 MiB.");
|
| 25 |
+
const manifest = JSON.parse(new TextDecoder().decode(bytes)) as BundleManifest;
|
| 26 |
+
if (manifest.schema_version !== "vons.bundle.manifest/v1" || !manifest.metadata
|
| 27 |
+
|| !["direct", "diffusion"].includes(manifest.metadata.backend)
|
| 28 |
+
|| !Array.isArray(manifest.files) || manifest.files.length > 2048) {
|
| 29 |
+
throw new Error("Choose a Vons v1 manifest for a full Direct or Diffusion ONNX graph.");
|
| 30 |
+
}
|
| 31 |
+
const paths = new Set<string>();
|
| 32 |
+
for (const file of manifest.files) {
|
| 33 |
+
if (!file || typeof file.path !== "string" || !safePath(file.path) || paths.has(file.path)
|
| 34 |
+
|| !Number.isSafeInteger(file.bytes) || file.bytes < 0 || file.bytes > MAX_BUNDLE_BYTES
|
| 35 |
+
|| !/^[0-9a-f]{64}$/.test(file.sha256)) throw new Error("Invalid manifest file record.");
|
| 36 |
+
paths.add(file.path);
|
| 37 |
+
}
|
| 38 |
+
const graphs = manifest.files.filter((f) => f.role === "model_graph");
|
| 39 |
+
const tokenizers = ["tokenizer.json", "tokenizer_config.json"].map((name) =>
|
| 40 |
+
manifest.files.filter((f) => f.role === "tokenizer" && (f.path === name || f.path.endsWith(`/${name}`))));
|
| 41 |
+
if (graphs.length !== 1 || tokenizers.some((files) => files.length !== 1)) {
|
| 42 |
+
throw new Error("The manifest must name one full graph and both tokenizer JSON files.");
|
| 43 |
+
}
|
| 44 |
+
const external = manifest.metadata.graph?.external_data_locations ?? [];
|
| 45 |
+
if (!Array.isArray(external) || external.some((path) => typeof path !== "string" || !safePath(path)
|
| 46 |
+
|| !manifest.files.some((f) => f.path === path && f.role === "model_external_data"))) {
|
| 47 |
+
throw new Error("The graph has an invalid external-data reference.");
|
| 48 |
+
}
|
| 49 |
+
return { manifest, paths: [...new Set([graphs[0].path, ...tokenizers.map((f) => f[0].path), ...external])] };
|
| 50 |
+
}
|
| 51 |
+
export async function verifyBundle(bundle: LocalBundle): Promise<void> {
|
| 52 |
+
if (!safePath(bundle.manifestPath) || await digest(bundle.manifestBytes) !== bundle.manifestHash) {
|
| 53 |
+
throw new Error("The saved manifest failed its integrity check. Import the model again.");
|
| 54 |
+
}
|
| 55 |
+
const { manifest, paths } = requiredFiles(bundle.manifestBytes);
|
| 56 |
+
if (manifest.metadata.backend !== bundle.backend || bundle.assets.length !== paths.length) {
|
| 57 |
+
throw new Error("Model bundle contents do not match the manifest.");
|
| 58 |
+
}
|
| 59 |
+
let total = bundle.manifestBytes.byteLength;
|
| 60 |
+
for (const path of paths) {
|
| 61 |
+
const matches = bundle.assets.filter((a) => a.path === path);
|
| 62 |
+
const record = manifest.files.find((f) => f.path === path)!;
|
| 63 |
+
if (matches.length !== 1 || matches[0].bytes.byteLength !== record.bytes) throw new Error(`Missing or wrong-sized model file: ${path}`);
|
| 64 |
+
total += record.bytes;
|
| 65 |
+
if (total > MAX_BUNDLE_BYTES) throw new Error("The selected runtime assets exceed 128 MiB.");
|
| 66 |
+
if (await digest(matches[0].bytes) !== record.sha256) throw new Error(`Model file failed SHA-256 verification: ${path}`);
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
export async function importBundle(files: File[]): Promise<LocalBundle> {
|
| 70 |
+
if (files.length === 0 || files.length > 2048) throw new Error("Choose one model folder with at most 2048 files.");
|
| 71 |
+
const manifests = files.filter((f) => f.name === "bundle-manifest-v1.json");
|
| 72 |
+
if (manifests.length !== 1) throw new Error("Choose the folder containing exactly one bundle-manifest-v1.json.");
|
| 73 |
+
const file = manifests[0];
|
| 74 |
+
if (file.size > 2 * 1024 * 1024) throw new Error("Manifest exceeds 2 MiB.");
|
| 75 |
+
const root = file.webkitRelativePath.slice(0, -file.name.length);
|
| 76 |
+
const manifestBytes = await file.arrayBuffer();
|
| 77 |
+
const { manifest, paths } = requiredFiles(manifestBytes);
|
| 78 |
+
const assets: LocalAsset[] = [];
|
| 79 |
+
let total = manifestBytes.byteLength;
|
| 80 |
+
for (const path of paths) {
|
| 81 |
+
const selected = files.filter((f) => f.webkitRelativePath === root + path);
|
| 82 |
+
const expected = manifest.files.find((f) => f.path === path)!;
|
| 83 |
+
if (selected.length !== 1 || selected[0].size !== expected.bytes) throw new Error(`Missing or wrong-sized model file: ${path}`);
|
| 84 |
+
total += selected[0].size;
|
| 85 |
+
if (total > MAX_BUNDLE_BYTES) throw new Error("The selected runtime assets exceed 128 MiB.");
|
| 86 |
+
assets.push({ path, bytes: await selected[0].arrayBuffer() });
|
| 87 |
+
}
|
| 88 |
+
const bundle: LocalBundle = { manifestPath: file.name, manifestBytes,
|
| 89 |
+
manifestHash: await digest(manifestBytes), backend: manifest.metadata.backend,
|
| 90 |
+
modelId: manifest.metadata.model_id ?? `Vons ${manifest.metadata.backend}`, assets };
|
| 91 |
+
await verifyBundle(bundle);
|
| 92 |
+
return bundle;
|
| 93 |
+
}
|
| 94 |
+
export function makeRequest(state: string, prompt: string, candidates: string): DecisionRequest {
|
| 95 |
+
const request: DecisionRequest = { state: state.trim(), seed: 7,
|
| 96 |
+
questions: [{ id: "decision", type: "choice", prompt: prompt.trim(),
|
| 97 |
+
options: candidates.split(/\r?\n/).map((s) => s.trim()).filter(Boolean) }] };
|
| 98 |
+
validateRequest(request);
|
| 99 |
+
return request;
|
| 100 |
+
}
|
sdk/typescript/extension/manifest.json
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"manifest_version": 3,
|
| 3 |
+
"name": "Vons — Local Decisions",
|
| 4 |
+
"version": "0.1.0",
|
| 5 |
+
"description": "Compare your candidate choices locally. Import a Vons model bundle; the host stays in control.",
|
| 6 |
+
"minimum_chrome_version": "114",
|
| 7 |
+
"permissions": ["sidePanel"],
|
| 8 |
+
"icons": {
|
| 9 |
+
"16": "icon-16.png",
|
| 10 |
+
"32": "icon-32.png",
|
| 11 |
+
"48": "icon-48.png",
|
| 12 |
+
"128": "icon-128.png"
|
| 13 |
+
},
|
| 14 |
+
"action": {"default_title": "Open Vons"},
|
| 15 |
+
"background": {"service_worker": "background.mjs", "type": "module"},
|
| 16 |
+
"side_panel": {"default_path": "panel.html"},
|
| 17 |
+
"content_security_policy": {
|
| 18 |
+
"extension_pages": "default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; object-src 'none'; connect-src 'self'; worker-src 'self'; style-src 'self'; img-src 'self'; frame-src 'none'; base-uri 'none'; form-action 'none'"
|
| 19 |
+
}
|
| 20 |
+
}
|
sdk/typescript/extension/panel.css
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
:root { color-scheme: light dark; --bg:#f5f6f1; --surface:#fff; --ink:#20251e; --muted:#697063; --line:#d7ded0; --accent:#c5ff7a; --error:#a63725; font:14px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; }
|
| 2 |
+
* { box-sizing:border-box; }
|
| 3 |
+
body { margin:0; background:var(--bg); color:var(--ink); min-width:280px; }
|
| 4 |
+
header { padding:23px 22px 19px; border-bottom:1px solid var(--line); display:flex; gap:12px; align-items:center; justify-content:space-between; }
|
| 5 |
+
header img { object-fit:contain; }
|
| 6 |
+
.version { font-size:9px; font-weight:650; letter-spacing:.08em; text-align:right; }
|
| 7 |
+
main { max-width:560px; margin:auto; padding:0 22px; }
|
| 8 |
+
.intro { padding:30px 0 24px; }
|
| 9 |
+
.eyebrow { font-size:10px; font-weight:700; letter-spacing:.12em; }
|
| 10 |
+
h1 { font-size:clamp(26px,7.8vw,38px); line-height:1.1; font-weight:620; letter-spacing:-.04em; margin:14px 0; }
|
| 11 |
+
p { margin:10px 0; }
|
| 12 |
+
.intro>p:last-child, .model>p { color:var(--muted); font-size:13px; }
|
| 13 |
+
.model { background:var(--surface); padding:18px; border:1px solid var(--line); border-radius:12px; }
|
| 14 |
+
.section-title { display:flex; align-items:center; gap:10px; justify-content:space-between; margin-bottom:13px; }
|
| 15 |
+
h2 { margin:0; font-size:11px; font-weight:450; white-space:nowrap; color:var(--muted); }
|
| 16 |
+
h2 span { color:var(--ink); font-size:13px; font-weight:650; margin-left:9px; }
|
| 17 |
+
.badge { background:var(--bg); border:1px solid var(--line); border-radius:4px; font-size:10px; padding:3px 7px; white-space:nowrap; }
|
| 18 |
+
button,input,textarea,select { font:inherit; }
|
| 19 |
+
button,a,.file-button,summary { -webkit-tap-highlight-color:transparent; }
|
| 20 |
+
button,.file-button { cursor:pointer; }
|
| 21 |
+
a { color:inherit; text-underline-offset:3px; }
|
| 22 |
+
button:disabled { opacity:.45; cursor:not-allowed; }
|
| 23 |
+
button { border:1px solid var(--line); border-radius:6px; padding:10px 13px; background:var(--surface); color:var(--ink); }
|
| 24 |
+
.file-button { display:flex; justify-content:space-between; background:var(--ink); color:var(--surface); padding:11px 13px; border-radius:6px; font-weight:600; margin:16px 0 12px; }
|
| 25 |
+
.remember { display:flex; gap:8px; align-items:center; font-size:11px; color:var(--muted); }
|
| 26 |
+
.remember input { accent-color:#537c29; margin:0; }
|
| 27 |
+
.model-actions { display:flex; gap:14px; align-items:center; justify-content:space-between; margin-top:10px; font-size:11px; }
|
| 28 |
+
.text-button { border:0; padding:3px 0; background:none; text-decoration:underline; text-underline-offset:3px; font-size:11px; }
|
| 29 |
+
form { padding:27px 0 20px; }
|
| 30 |
+
form>label { display:block; margin:17px 0 6px; font-size:12px; font-weight:650; }
|
| 31 |
+
.hint { color:var(--muted); font-size:11px; font-weight:400; }
|
| 32 |
+
label .hint { float:right; }
|
| 33 |
+
textarea,form>input,select { width:100%; border:1px solid var(--line); border-radius:6px; background:var(--surface); color:var(--ink); padding:11px 12px; }
|
| 34 |
+
textarea { resize:vertical; min-height:70px; }
|
| 35 |
+
textarea::placeholder,input::placeholder { color:#818879; }
|
| 36 |
+
:focus-visible { outline:2px solid #679827; outline-offset:3px; }
|
| 37 |
+
details { border-top:1px solid var(--line); margin-top:17px; padding:12px 0 0; font-size:12px; }
|
| 38 |
+
summary { cursor:pointer; color:var(--muted); }
|
| 39 |
+
details label { display:block; margin:10px 0 5px; }
|
| 40 |
+
.run-actions { display:flex; gap:10px; margin-top:19px; }
|
| 41 |
+
#run { flex:1; background:var(--accent); color:#1e2915; font-weight:650; border-color:transparent; display:flex; justify-content:space-between; }
|
| 42 |
+
#status { font-size:12px; color:var(--muted); overflow-wrap:anywhere; }
|
| 43 |
+
#status[data-error="true"] { color:var(--error); }
|
| 44 |
+
.result { border-top:2px solid var(--ink); padding:23px 0; }
|
| 45 |
+
.result h3 { font-size:24px; line-height:1.25; margin:9px 0; overflow-wrap:anywhere; }
|
| 46 |
+
#reason { color:var(--muted); font-size:12px; }
|
| 47 |
+
ol { list-style:none; padding:0; margin:18px 0; }
|
| 48 |
+
li { margin:12px 0; }
|
| 49 |
+
.rank-label { display:flex; justify-content:space-between; gap:15px; font-size:12px; overflow-wrap:anywhere; }
|
| 50 |
+
progress { width:100%; height:5px; border:0; display:block; margin-top:7px; accent-color:#709b42; }
|
| 51 |
+
progress::-webkit-progress-bar { background:var(--line); border-radius:4px; }
|
| 52 |
+
progress::-webkit-progress-value { background:#709b42; border-radius:4px; }
|
| 53 |
+
.boundary { font-size:11px; background:var(--surface); border-left:3px solid #a5b592; padding:9px 12px; }
|
| 54 |
+
pre { font:10px/1.6 ui-monospace,monospace; overflow:auto; max-height:240px; white-space:pre-wrap; overflow-wrap:anywhere; }
|
| 55 |
+
footer { border-top:1px solid var(--line); padding:16px 0 25px; color:var(--muted); font-size:10px; }
|
| 56 |
+
footer p:first-child { color:var(--ink); font-weight:650; }
|
| 57 |
+
nav { display:flex; gap:18px; margin-top:15px; }
|
| 58 |
+
[hidden] { display:none!important; }
|
| 59 |
+
.visually-hidden { position:absolute; width:1px; height:1px; overflow:hidden; clip-path:inset(50%); }
|
| 60 |
+
.file-button:has(+input:focus-visible) { outline:2px solid #679827; outline-offset:3px; }
|
| 61 |
+
@media(prefers-color-scheme:dark) { :root { --bg:#1d221c; --surface:#252c23; --ink:#edf4e8; --muted:#a5b09e; --line:#434e3d; --error:#ffb09c; } header img { background:#f5f6f1; border-radius:2px; } }
|
sdk/typescript/extension/panel.html
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<title>Vons — Local Decisions</title>
|
| 7 |
+
<link rel="stylesheet" href="panel.css">
|
| 8 |
+
</head>
|
| 9 |
+
<body>
|
| 10 |
+
<header>
|
| 11 |
+
<img src="brand.png" alt="INLEVEL9" width="132" height="23">
|
| 12 |
+
<span class="version">RESEARCH PREVIEW</span>
|
| 13 |
+
</header>
|
| 14 |
+
<main>
|
| 15 |
+
<div class="intro"><p class="eyebrow">YOUR OPTIONS. YOUR CONTROL.</p><h1>A little help<br>with the next choice.</h1><p>Compare a few candidates with a model on your device. You decide what happens next.</p></div>
|
| 16 |
+
<section class="model" aria-labelledby="model-heading">
|
| 17 |
+
<div class="section-title"><h2 id="model-heading">01 <span>Choose a model</span></h2><span id="model-badge" class="badge">Not loaded</span></div>
|
| 18 |
+
<p id="model-description">Import a Vons ONNX bundle folder. Model weights are supplied separately.</p>
|
| 19 |
+
<label class="file-button" for="bundle-files">Import model folder <span aria-hidden="true">↗</span></label>
|
| 20 |
+
<input id="bundle-files" type="file" webkitdirectory multiple class="visually-hidden">
|
| 21 |
+
<label class="remember"><input id="remember" type="checkbox"> Keep this model on this device</label>
|
| 22 |
+
<div class="model-actions"><button id="forget" type="button" class="text-button" disabled>Remove model</button><a href="https://huggingface.co/INLEVEL9/Vons" target="_blank" rel="noreferrer">Bundle guide ↗</a></div>
|
| 23 |
+
</section>
|
| 24 |
+
<form id="decision-form">
|
| 25 |
+
<div class="section-title"><h2>02 <span>Frame your decision</span></h2><button id="example" type="button" class="text-button">Use example</button></div>
|
| 26 |
+
<label for="state">Context</label><textarea id="state" rows="3" maxlength="20000" placeholder="What does the model need to know?" required></textarea>
|
| 27 |
+
<label for="question">Question</label><input id="question" maxlength="2000" placeholder="Which option fits this situation?" required>
|
| 28 |
+
<label for="options">Candidates <span class="hint">One per line · 2–32</span></label><textarea id="options" rows="4" maxlength="20000" placeholder="First option Second option" required></textarea>
|
| 29 |
+
<details><summary>Runtime settings</summary><label for="provider">Execution provider</label><select id="provider"><option value="wasm">WASM · CPU (default)</option value="webgpu">WebGPU · experimental</option></select><p class="hint">WebGPU requires a compatible adapter. Failed requests stay visible; no silent fallback.</p></details>
|
| 30 |
+
<div class="run-actions"><button id="run" type="submit" disabled>Compare candidates <span aria-hidden="true">→</span></button><button id="cancel" type="button" hidden>Cancel</button></div>
|
| 31 |
+
<p id="status" role="status" aria-live="polite">Start by importing a model. Prompts stay in this panel.</p>
|
| 32 |
+
</form>
|
| 33 |
+
<section id="result" class="result" aria-labelledby="result-heading" hidden>
|
| 34 |
+
<div class="section-title"><h2 id="result-heading">03 <span>Review the proposal</span></h2><button id="copy" type="button" class="text-button">Copy JSON</button></div>
|
| 35 |
+
<p id="decision-label" class="eyebrow"></p><h3 id="choice"></h3><p id="reason"></p>
|
| 36 |
+
<ol id="ranking"></ol><p id="timing" class="hint"></p>
|
| 37 |
+
<p class="boundary">Scores are uncalibrated model outputs. A proposal is not permission to act.</p>
|
| 38 |
+
<details><summary>Response and provenance</summary><pre id="raw"></pre></details>
|
| 39 |
+
</section>
|
| 40 |
+
<footer><p>Local processing. No telemetry. No page access.</p><p>Text and decisions are not saved. Closing the panel ends the session. Only a model you choose to keep is stored locally.</p><nav><a href="privacy.html" target="_blank">Privacy & terms</a><a href="https://huggingface.co/INLEVEL9/Vons/tree/main/docs/publications" target="_blank" rel="noreferrer">Research ↗</a></nav></footer>
|
| 41 |
+
</main>
|
| 42 |
+
<script src="panel.js" type="module"></script>
|
| 43 |
+
</body>
|
| 44 |
+
</html>
|
sdk/typescript/extension/panel.ts
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { importBundle, verifyBundle, makeRequest, type LocalBundle } from "./bundle.ts";
|
| 2 |
+
import { modelStorage } from "./storage.ts";
|
| 3 |
+
import { validateResponse, type DecisionResponse } from "../src/index.ts";
|
| 4 |
+
|
| 5 |
+
function element<T extends HTMLElement>(id: string): T {
|
| 6 |
+
const node = document.getElementById(id);
|
| 7 |
+
if (!node) throw new Error(`Missing panel element: ${id}`);
|
| 8 |
+
return node as T;
|
| 9 |
+
}
|
| 10 |
+
const files = element<HTMLInputElement>("bundle-files");
|
| 11 |
+
const remember = element<HTMLInputElement>("remember");
|
| 12 |
+
const run = element<HTMLButtonElement>("run");
|
| 13 |
+
const cancel = element<HTMLButtonElement>("cancel");
|
| 14 |
+
const forget = element<HTMLButtonElement>("forget");
|
| 15 |
+
const provider = element<HTMLSelectElement>("provider");
|
| 16 |
+
const form = element<HTMLFormElement>("decision-form");
|
| 17 |
+
let bundle: LocalBundle | undefined;
|
| 18 |
+
let worker: Worker | undefined;
|
| 19 |
+
let workerKey = "";
|
| 20 |
+
let busy = false;
|
| 21 |
+
let importing = false;
|
| 22 |
+
let lastResult = "";
|
| 23 |
+
let timeout: ReturnType<typeof setTimeout> | undefined;
|
| 24 |
+
|
| 25 |
+
function status(text: string, error = false): void {
|
| 26 |
+
const node = element("status"); node.textContent = text; node.dataset.error = String(error);
|
| 27 |
+
}
|
| 28 |
+
function controls(): void {
|
| 29 |
+
run.disabled = busy || importing || !bundle;
|
| 30 |
+
forget.disabled = busy || importing || !bundle;
|
| 31 |
+
files.disabled = busy || importing;
|
| 32 |
+
remember.disabled = busy || importing;
|
| 33 |
+
provider.disabled = busy || importing;
|
| 34 |
+
for (const id of ["state", "question", "options", "example"]) element<HTMLInputElement | HTMLButtonElement>(id).disabled = busy;
|
| 35 |
+
cancel.hidden = !busy;
|
| 36 |
+
form.setAttribute("aria-busy", String(busy));
|
| 37 |
+
}
|
| 38 |
+
function stop(): void {
|
| 39 |
+
worker?.terminate(); worker = undefined; workerKey = "";
|
| 40 |
+
clearTimeout(timeout); busy = false; controls();
|
| 41 |
+
}
|
| 42 |
+
function modelLabel(): void {
|
| 43 |
+
element("model-badge").textContent = bundle ? bundle.backend : "Not loaded";
|
| 44 |
+
element("model-description").textContent = bundle
|
| 45 |
+
? `${bundle.modelId} · ${(bundle.assets.reduce((sum, a) => sum + a.bytes.byteLength, 0) / 1048576).toFixed(1)} MiB · SHA-256 checked`
|
| 46 |
+
: "Import a Vons ONNX bundle folder. Model weights are supplied separately.";
|
| 47 |
+
controls();
|
| 48 |
+
}
|
| 49 |
+
function errorMessage(error: unknown): string { return error instanceof Error ? error.message : "The operation failed."; }
|
| 50 |
+
function showResult(data: { response: DecisionResponse; manifestHash: string; runtime: unknown; loadMs: number; inferenceMs: number }): void {
|
| 51 |
+
validateResponse(data.response);
|
| 52 |
+
const answer = data.response.answers[0];
|
| 53 |
+
if (!answer) throw new Error("The model returned no answer.");
|
| 54 |
+
element("decision-label").textContent = answer.status === "abstain" ? "NO RECOMMENDATION" : "MODEL PROPOSAL";
|
| 55 |
+
element("choice").textContent = answer.choice ?? "The model abstained.";
|
| 56 |
+
element("reason").textContent = answer.abstain_reason?.replaceAll("_", " ") ?? "Review this choice yourself before taking any action.";
|
| 57 |
+
const ranking = element("ranking"); ranking.replaceChildren();
|
| 58 |
+
for (const entry of [...answer.probabilities].sort((a, b) => b.probability - a.probability)) {
|
| 59 |
+
const item = document.createElement("li"); const row = document.createElement("div"); row.className = "rank-label";
|
| 60 |
+
const label = document.createElement("span"); label.textContent = entry.option;
|
| 61 |
+
const value = document.createElement("span"); value.textContent = `${(entry.probability * 100).toFixed(1)}%`;
|
| 62 |
+
const bar = document.createElement("progress"); bar.max = 1; bar.value = entry.probability; bar.setAttribute("aria-label", `${entry.option} model score`);
|
| 63 |
+
row.append(label, value); item.append(row, bar); ranking.append(item);
|
| 64 |
+
}
|
| 65 |
+
element("timing").textContent = `Local run: ${data.inferenceMs.toFixed(0)} ms · load/check: ${data.loadMs.toFixed(0)} ms. This is one run, not a benchmark.`;
|
| 66 |
+
lastResult = JSON.stringify(data, null, 2); element("raw").textContent = lastResult;
|
| 67 |
+
element("result").hidden = false;
|
| 68 |
+
}
|
| 69 |
+
files.addEventListener("change", async () => {
|
| 70 |
+
if (!files.files?.length) return;
|
| 71 |
+
importing = true; stop(); controls(); element("result").hidden = true;
|
| 72 |
+
status("Reading and verifying the selected model files…");
|
| 73 |
+
try {
|
| 74 |
+
const next = await importBundle(Array.from(files.files));
|
| 75 |
+
// Do not make a new model active until its requested storage operation succeeds.
|
| 76 |
+
if (remember.checked) await modelStorage("write", next);
|
| 77 |
+
else await modelStorage("clear");
|
| 78 |
+
bundle = next; status("Model ready. Add context and candidates, then compare.");
|
| 79 |
+
} catch (error) { status(errorMessage(error), true); }
|
| 80 |
+
finally { importing = false; files.value = ""; modelLabel(); }
|
| 81 |
+
});
|
| 82 |
+
remember.addEventListener("change", async () => {
|
| 83 |
+
importing = true; controls();
|
| 84 |
+
try {
|
| 85 |
+
if (remember.checked && bundle) await modelStorage("write", bundle);
|
| 86 |
+
else await modelStorage("clear");
|
| 87 |
+
status(remember.checked ? "This model will be available next time. Text is never saved." : "Stored model removed. The current model remains in this session only.");
|
| 88 |
+
} catch (error) { remember.checked = !remember.checked; status(errorMessage(error), true); }
|
| 89 |
+
finally { importing = false; controls(); }
|
| 90 |
+
});
|
| 91 |
+
forget.addEventListener("click", async () => {
|
| 92 |
+
importing = true; stop(); controls();
|
| 93 |
+
try {
|
| 94 |
+
await modelStorage("clear"); bundle = undefined; remember.checked = false;
|
| 95 |
+
element("result").hidden = true; lastResult = ""; element("raw").textContent = "";
|
| 96 |
+
status("Model removed from this device.");
|
| 97 |
+
} catch (error) { status(errorMessage(error), true); }
|
| 98 |
+
finally { importing = false; modelLabel(); }
|
| 99 |
+
});
|
| 100 |
+
element("example").addEventListener("click", () => {
|
| 101 |
+
element<HTMLInputElement>("state").value = "A document has unsaved edits. The author wants to keep the changes before closing it.";
|
| 102 |
+
element<HTMLInputElement>("question").value = "What should happen next?";
|
| 103 |
+
element<HTMLInputElement>("options").value = "Save the document\nDiscard the changes\nAsk the author for more context";
|
| 104 |
+
element("result").hidden = true;
|
| 105 |
+
status("Example loaded. This is an illustrative input, not a quality test.");
|
| 106 |
+
});
|
| 107 |
+
form.addEventListener("submit", (event) => {
|
| 108 |
+
event.preventDefault(); if (busy || importing || !bundle) return;
|
| 109 |
+
try {
|
| 110 |
+
const request = makeRequest(element<HTMLInputElement>("state").value, element<HTMLInputElement>("question").value, element<HTMLInputElement>("options").value);
|
| 111 |
+
const selected = provider.value === "webgpu" ? "webgpu" : "wasm";
|
| 112 |
+
const key = `${bundle.manifestHash}:${selected}`;
|
| 113 |
+
if (workerKey !== key) stop();
|
| 114 |
+
if (!worker) {
|
| 115 |
+
worker = new Worker(new URL("worker.js", import.meta.url), { type: "module" });
|
| 116 |
+
worker.onmessage = (message) => {
|
| 117 |
+
const data = message.data;
|
| 118 |
+
if (data.type === "progress") { status(data.text); return; }
|
| 119 |
+
clearTimeout(timeout); busy = false; controls();
|
| 120 |
+
if (data.type === "error") { stop(); status(data.text, true); return; }
|
| 121 |
+
try { showResult(data); status("Comparison complete. No page action was taken."); }
|
| 122 |
+
catch (error) { stop(); status(errorMessage(error), true); }
|
| 123 |
+
};
|
| 124 |
+
worker.onerror = () => { stop(); status("The local runtime stopped unexpectedly. You can try again with WASM.", true); };
|
| 125 |
+
}
|
| 126 |
+
busy = true; controls(); element("result").hidden = true;
|
| 127 |
+
status("Starting the local comparison…");
|
| 128 |
+
worker.postMessage({ request, provider: selected, ...(workerKey === key ? {} : { bundle }) }); workerKey = key;
|
| 129 |
+
timeout = setTimeout(() => { stop(); status("The comparison exceeded two minutes and was stopped. Try fewer or shorter candidates.", true); }, 120000);
|
| 130 |
+
} catch (error) { stop(); status(errorMessage(error), true); }
|
| 131 |
+
});
|
| 132 |
+
cancel.addEventListener("click", () => { stop(); status("Canceled. No decision was returned. The model can be used again."); });
|
| 133 |
+
element("copy").addEventListener("click", async () => {
|
| 134 |
+
try { await navigator.clipboard.writeText(lastResult); status("Response JSON copied. It contains your candidate text."); }
|
| 135 |
+
catch { status("Clipboard access was unavailable. Select the JSON text to copy it manually.", true); }
|
| 136 |
+
});
|
| 137 |
+
window.addEventListener("pagehide", stop);
|
| 138 |
+
async function restore(): Promise<void> {
|
| 139 |
+
importing = true; controls();
|
| 140 |
+
try {
|
| 141 |
+
const saved = await modelStorage("read");
|
| 142 |
+
if (saved) { await verifyBundle(saved); bundle = saved; remember.checked = true; status("Your saved model is ready. Context and decisions were not retained."); }
|
| 143 |
+
} catch (error) { status(`${errorMessage(error)} You can import a model folder again.`, true); }
|
| 144 |
+
finally { importing = false; modelLabel(); }
|
| 145 |
+
}
|
| 146 |
+
void restore();
|
sdk/typescript/extension/privacy.html
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Vons — Privacy and terms</title><link rel="stylesheet" href="panel.css"></head><body><main>
|
| 3 |
+
<div class="intro"><p class="eyebrow">VONS · RESEARCH PREVIEW 0.1.0</p><h1>Local by design.</h1><p>The extension compares candidate choices using a model you import. It cannot read or modify web pages and never executes a proposed action.</p></div>
|
| 4 |
+
<h2><span>What is processed</span></h2><p>Only the context, question, candidates and model files you explicitly provide. Inference runs on this device using packaged ONNX Runtime Web code. There are no server uploads, analytics, telemetry, account requirements or remote model downloads.</p>
|
| 5 |
+
<h2><span>What is retained</span></h2><p>Context and results stay in the open panel's memory. Closing the panel ends that session. If you choose “Keep this model on this device,” model assets and their manifest are stored in local IndexedDB, not Chrome Sync. Remove model clears that stored model. Turning off the option also clears the stored copy.</p>
|
| 6 |
+
<h2><span>Permissions</span></h2><p>The sole extension permission is sidePanel, used to display Vons beside the current tab. No page, cookie, history, clipboard-read, background network or browsing-content access is requested. Copy JSON writes only the displayed result to your clipboard after you click it. Model file access comes from the folder picker you open.</p>
|
| 7 |
+
<h2><span>External links</span></h2><p>Research and bundle-guide links open Hugging Face only when clicked. That destination has its own privacy policy. Context, candidates and model contents are not included in those links.</p>
|
| 8 |
+
<h2><span>Limits and terms</span></h2><p>This is research software. Model scores are not calibrated probabilities of correctness. A proposal does not establish quality, consent or permission to act. The current publication reports negative external evaluation results and unresolved deployment gates. Model weights are not included; review their origin and redistribution terms separately.</p><p>Vons uses the <a href="LICENSE">Vons Community and Commercial License 1.0</a>: qualifying noncommercial use is free; enterprise and commercial use require a separate written agreement. Third-party code retains its own terms in <a href="THIRD_PARTY_NOTICES.txt">bundled notices</a>. Articles use CC BY 4.0 separately.</p>
|
| 9 |
+
<footer><p>Contact: <a href="mailto:oswarld@inlevel9.com">oswarld@inlevel9.com</a> · INLEVEL9</p><p>Updated September 25, 2026.</p></footer>
|
| 10 |
+
</main></body></html>
|
sdk/typescript/extension/storage.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import type { LocalBundle } from "./bundle.ts";
|
| 2 |
+
const DATABASE = "vons-local-model";
|
| 3 |
+
async function open(): Promise<IDBDatabase> {
|
| 4 |
+
return new Promise((resolve, reject) => {
|
| 5 |
+
const request = indexedDB.open(DATABASE, 1);
|
| 6 |
+
request.onupgradeneeded = () => request.result.createObjectStore("model");
|
| 7 |
+
request.onsuccess = () => resolve(request.result);
|
| 8 |
+
request.onerror = () => reject(new Error("Local model storage is unavailable."));
|
| 9 |
+
request.onblocked = () => reject(new Error("Close other Vons panels before updating storage."));
|
| 10 |
+
});
|
| 11 |
+
}
|
| 12 |
+
export async function modelStorage(action: "read" | "write" | "clear", model?: LocalBundle): Promise<LocalBundle | undefined> {
|
| 13 |
+
const db = await open();
|
| 14 |
+
try {
|
| 15 |
+
return await new Promise((resolve, reject) => {
|
| 16 |
+
const transaction = db.transaction("model", action === "read" ? "readonly" : "readwrite");
|
| 17 |
+
const store = transaction.objectStore("model");
|
| 18 |
+
const request = action === "read" ? store.get("active") : action === "write" ? store.put(model, "active") : store.clear();
|
| 19 |
+
transaction.oncomplete = () => resolve(action === "read" ? request.result as LocalBundle | undefined : undefined);
|
| 20 |
+
transaction.onabort = transaction.onerror = () => reject(new Error("Could not update local model storage. Check available space."));
|
| 21 |
+
});
|
| 22 |
+
} finally { db.close(); }
|
| 23 |
+
}
|
sdk/typescript/extension/tsconfig.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"compilerOptions": {
|
| 3 |
+
"target": "ES2022",
|
| 4 |
+
"module": "ESNext",
|
| 5 |
+
"moduleResolution": "Bundler",
|
| 6 |
+
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
| 7 |
+
"strict": true,
|
| 8 |
+
"noEmit": true,
|
| 9 |
+
"allowImportingTsExtensions": true,
|
| 10 |
+
"skipLibCheck": true
|
| 11 |
+
},
|
| 12 |
+
"include": ["*.ts", "../src/*.ts"]
|
| 13 |
+
}
|
sdk/typescript/extension/worker.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { createOnnxWebBackend, type OnnxWebBackend } from "../src/onnx.ts";
|
| 2 |
+
import type { DecisionRequest } from "../src/index.ts";
|
| 3 |
+
import { verifyBundle, type LocalBundle } from "./bundle.ts";
|
| 4 |
+
|
| 5 |
+
let backend: OnnxWebBackend | undefined;
|
| 6 |
+
let loadedKey = "";
|
| 7 |
+
self.onmessage = async (event: MessageEvent<{ bundle?: LocalBundle; provider: "wasm" | "webgpu"; request: DecisionRequest }>) => {
|
| 8 |
+
const started = performance.now();
|
| 9 |
+
try {
|
| 10 |
+
const { bundle, provider, request } = event.data;
|
| 11 |
+
if (bundle) {
|
| 12 |
+
const key = `${bundle.manifestHash}:${provider}`;
|
| 13 |
+
if (key !== loadedKey) {
|
| 14 |
+
await backend?.dispose(); backend = undefined; loadedKey = "";
|
| 15 |
+
self.postMessage({ type: "progress", text: "Checking model files and starting the local runtime…" });
|
| 16 |
+
await verifyBundle(bundle);
|
| 17 |
+
// This synthetic origin is only a lookup key. No model request can use network fetch.
|
| 18 |
+
const base = "https://vons-bundle.invalid/";
|
| 19 |
+
const bytes = new Map(bundle.assets.map((f) => [new URL(f.path, base).href, f.bytes]));
|
| 20 |
+
bytes.set(new URL(bundle.manifestPath, base).href, bundle.manifestBytes);
|
| 21 |
+
const localFetch: typeof fetch = async (input) => {
|
| 22 |
+
const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
|
| 23 |
+
const asset = bytes.get(url);
|
| 24 |
+
if (!asset) throw new Error("Requested model asset is not in the selected local bundle.");
|
| 25 |
+
return new Response(asset);
|
| 26 |
+
};
|
| 27 |
+
backend = await createOnnxWebBackend({ manifestUrl: new URL(bundle.manifestPath, base),
|
| 28 |
+
expectedManifestSha256: bundle.manifestHash, fetch: localFetch, provider,
|
| 29 |
+
wasmPaths: new URL("runtime/", self.location.href).href, wasmNumThreads: 1 });
|
| 30 |
+
loadedKey = key;
|
| 31 |
+
}
|
| 32 |
+
}
|
| 33 |
+
if (!backend || backend.runtimeInfo.provider !== provider) throw new Error("Import or reload the model before running.");
|
| 34 |
+
const ready = performance.now();
|
| 35 |
+
self.postMessage({ type: "progress", text: "Comparing your candidates on this device…" });
|
| 36 |
+
const response = await backend.decide(request);
|
| 37 |
+
self.postMessage({ type: "result", response, manifestHash: backend.manifestHash,
|
| 38 |
+
runtime: backend.runtimeInfo, loadMs: ready - started, inferenceMs: performance.now() - ready });
|
| 39 |
+
} catch (error) {
|
| 40 |
+
self.postMessage({ type: "error", text: error instanceof Error ? error.message : "Local inference failed." });
|
| 41 |
+
}
|
| 42 |
+
};
|
sdk/typescript/package-lock.json
CHANGED
|
@@ -7,17 +7,18 @@
|
|
| 7 |
"": {
|
| 8 |
"name": "@vons/decision-sdk",
|
| 9 |
"version": "0.1.0",
|
|
|
|
| 10 |
"dependencies": {
|
| 11 |
"@huggingface/tokenizers": "0.2.0",
|
| 12 |
"onnxruntime-web": "1.30.0"
|
| 13 |
},
|
| 14 |
"devDependencies": {
|
| 15 |
-
"esbuild": "0.28.2"
|
|
|
|
| 16 |
},
|
| 17 |
"engines": {
|
| 18 |
"node": ">=22"
|
| 19 |
-
}
|
| 20 |
-
"license": "SEE LICENSE IN LICENSE"
|
| 21 |
},
|
| 22 |
"node_modules/@esbuild/aix-ppc64": {
|
| 23 |
"version": "0.28.2",
|
|
@@ -642,6 +643,20 @@
|
|
| 642 |
"node": ">=12.0.0"
|
| 643 |
}
|
| 644 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 645 |
"node_modules/undici-types": {
|
| 646 |
"version": "8.9.0",
|
| 647 |
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
|
|
|
|
| 7 |
"": {
|
| 8 |
"name": "@vons/decision-sdk",
|
| 9 |
"version": "0.1.0",
|
| 10 |
+
"license": "SEE LICENSE IN LICENSE",
|
| 11 |
"dependencies": {
|
| 12 |
"@huggingface/tokenizers": "0.2.0",
|
| 13 |
"onnxruntime-web": "1.30.0"
|
| 14 |
},
|
| 15 |
"devDependencies": {
|
| 16 |
+
"esbuild": "0.28.2",
|
| 17 |
+
"typescript": "5.9.3"
|
| 18 |
},
|
| 19 |
"engines": {
|
| 20 |
"node": ">=22"
|
| 21 |
+
}
|
|
|
|
| 22 |
},
|
| 23 |
"node_modules/@esbuild/aix-ppc64": {
|
| 24 |
"version": "0.28.2",
|
|
|
|
| 643 |
"node": ">=12.0.0"
|
| 644 |
}
|
| 645 |
},
|
| 646 |
+
"node_modules/typescript": {
|
| 647 |
+
"version": "5.9.3",
|
| 648 |
+
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
| 649 |
+
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
| 650 |
+
"dev": true,
|
| 651 |
+
"license": "Apache-2.0",
|
| 652 |
+
"bin": {
|
| 653 |
+
"tsc": "bin/tsc",
|
| 654 |
+
"tsserver": "bin/tsserver"
|
| 655 |
+
},
|
| 656 |
+
"engines": {
|
| 657 |
+
"node": ">=14.17"
|
| 658 |
+
}
|
| 659 |
+
},
|
| 660 |
"node_modules/undici-types": {
|
| 661 |
"version": "8.9.0",
|
| 662 |
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
|
sdk/typescript/package.json
CHANGED
|
@@ -12,16 +12,19 @@
|
|
| 12 |
"./onnx": "./src/onnx.ts"
|
| 13 |
},
|
| 14 |
"scripts": {
|
| 15 |
-
"test": "node --experimental-strip-types --test test/index.test.mjs test/onnx.test.mjs test/parity.test.mjs",
|
| 16 |
"build:browser": "esbuild demo/main.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/main.js",
|
| 17 |
"build:benchmark": "esbuild demo/benchmark.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/benchmark.js",
|
| 18 |
-
"build:parity": "esbuild demo/parity.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/parity.js"
|
|
|
|
|
|
|
| 19 |
},
|
| 20 |
"engines": {
|
| 21 |
"node": ">=22"
|
| 22 |
},
|
| 23 |
"devDependencies": {
|
| 24 |
-
"esbuild": "0.28.2"
|
|
|
|
| 25 |
},
|
| 26 |
"private": true,
|
| 27 |
"files": [
|
|
|
|
| 12 |
"./onnx": "./src/onnx.ts"
|
| 13 |
},
|
| 14 |
"scripts": {
|
| 15 |
+
"test": "node --experimental-strip-types --test test/index.test.mjs test/onnx.test.mjs test/parity.test.mjs test/extension.test.mjs",
|
| 16 |
"build:browser": "esbuild demo/main.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/main.js",
|
| 17 |
"build:benchmark": "esbuild demo/benchmark.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/benchmark.js",
|
| 18 |
+
"build:parity": "esbuild demo/parity.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/parity.js",
|
| 19 |
+
"build:extension": "node extension/build.mjs",
|
| 20 |
+
"typecheck:extension": "tsc -p extension/tsconfig.json"
|
| 21 |
},
|
| 22 |
"engines": {
|
| 23 |
"node": ">=22"
|
| 24 |
},
|
| 25 |
"devDependencies": {
|
| 26 |
+
"esbuild": "0.28.2",
|
| 27 |
+
"typescript": "5.9.3"
|
| 28 |
},
|
| 29 |
"private": true,
|
| 30 |
"files": [
|
sdk/typescript/src/onnx.ts
CHANGED
|
@@ -325,9 +325,20 @@ export function tokenizeCandidates(
|
|
| 325 |
state: DecisionRequest["state"],
|
| 326 |
question: Question,
|
| 327 |
options: readonly string[],
|
| 328 |
-
|
| 329 |
-
): TokenizedCandidate[] {
|
| 330 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
}
|
| 332 |
|
| 333 |
function padCandidates(candidates: readonly TokenizedCandidate[], slots: number, sequenceLength: number, paddingId: number): {
|
|
@@ -499,6 +510,7 @@ export class OnnxWebBackend implements DecisionBackend {
|
|
| 499 |
if (this.disposed) throw new Error("ONNX Runtime session has been disposed");
|
| 500 |
validateRequest(request);
|
| 501 |
const answers: QuestionAnswer[] = [];
|
|
|
|
| 502 |
for (const [questionIndex, question] of request.questions.entries()) {
|
| 503 |
const requestStarted = performance.now();
|
| 504 |
if (question.type === "score") {
|
|
@@ -510,18 +522,18 @@ export class OnnxWebBackend implements DecisionBackend {
|
|
| 510 |
throw new RangeError(`question ${question.id} has unsupported candidate count`);
|
| 511 |
}
|
| 512 |
const aggregateTokenCount = this.tokenizer.encode(candidateListText(request.state, question, options)).ids.length;
|
| 513 |
-
if (aggregateTokenCount >
|
| 514 |
-
throw new RangeError(`question ${question.id} exceeds the ${
|
| 515 |
}
|
| 516 |
const tokenizeStarted = performance.now();
|
| 517 |
-
const encoded = tokenizeCandidates(this.tokenizer, request.state, question, options,
|
| 518 |
const tokenized = performance.now();
|
| 519 |
-
const padded = padCandidates(encoded, slots,
|
| 520 |
let traceNoise: Float32Array | null = null;
|
| 521 |
const feeds: Record<string, unknown> = {
|
| 522 |
-
input_ids: tensor(this.runtime, "int64", padded.inputIds, [1, slots,
|
| 523 |
-
attention_mask: tensor(this.runtime, "int64", padded.attentionMask, [1, slots,
|
| 524 |
-
token_type_ids: tensor(this.runtime, "int64", padded.tokenTypeIds, [1, slots,
|
| 525 |
option_mask: tensor(this.runtime, "bool", padded.optionMask, [1, slots]),
|
| 526 |
};
|
| 527 |
if (this.backend === "diffusion") {
|
|
@@ -570,7 +582,7 @@ export class OnnxWebBackend implements DecisionBackend {
|
|
| 570 |
live_candidates: encoded.length,
|
| 571 |
allocated_candidates: slots,
|
| 572 |
live_tokens: encoded.reduce((total, candidate) => total + candidate.inputIds.length, 0),
|
| 573 |
-
sequence_length:
|
| 574 |
tokenizeMs: tokenized - tokenizeStarted,
|
| 575 |
prepareFeedMs: inferenceStarted - tokenized,
|
| 576 |
inferenceAndReadbackMs: readbackCompleted - inferenceStarted,
|
|
|
|
| 325 |
state: DecisionRequest["state"],
|
| 326 |
question: Question,
|
| 327 |
options: readonly string[],
|
| 328 |
+
sequenceBudget: number,
|
| 329 |
+
): { candidates: TokenizedCandidate[]; liveSequenceLength: number } {
|
| 330 |
+
if (!Number.isInteger(sequenceBudget) || sequenceBudget < 1 || sequenceBudget > 512) {
|
| 331 |
+
throw new RangeError("sequenceBudget must be an integer in [1, 512]");
|
| 332 |
+
}
|
| 333 |
+
const candidates = options.map((option) => encodeCandidate(tokenizer, candidateText(state, question, option), sequenceBudget));
|
| 334 |
+
let longest = 0;
|
| 335 |
+
for (const candidate of candidates) {
|
| 336 |
+
const length = candidate.inputIds.length;
|
| 337 |
+
if (length > longest) longest = length;
|
| 338 |
+
}
|
| 339 |
+
if (longest < 1) throw new Error("each tokenized candidate must contain at least one token");
|
| 340 |
+
const liveSequenceLength = Math.min(longest, sequenceBudget);
|
| 341 |
+
return { candidates, liveSequenceLength };
|
| 342 |
}
|
| 343 |
|
| 344 |
function padCandidates(candidates: readonly TokenizedCandidate[], slots: number, sequenceLength: number, paddingId: number): {
|
|
|
|
| 510 |
if (this.disposed) throw new Error("ONNX Runtime session has been disposed");
|
| 511 |
validateRequest(request);
|
| 512 |
const answers: QuestionAnswer[] = [];
|
| 513 |
+
const sequenceBudget = this.sequenceLength;
|
| 514 |
for (const [questionIndex, question] of request.questions.entries()) {
|
| 515 |
const requestStarted = performance.now();
|
| 516 |
if (question.type === "score") {
|
|
|
|
| 522 |
throw new RangeError(`question ${question.id} has unsupported candidate count`);
|
| 523 |
}
|
| 524 |
const aggregateTokenCount = this.tokenizer.encode(candidateListText(request.state, question, options)).ids.length;
|
| 525 |
+
if (aggregateTokenCount > sequenceBudget) {
|
| 526 |
+
throw new RangeError(`question ${question.id} exceeds the ${sequenceBudget}-token aggregate input budget (tokens=${aggregateTokenCount})`);
|
| 527 |
}
|
| 528 |
const tokenizeStarted = performance.now();
|
| 529 |
+
const { candidates: encoded, liveSequenceLength } = tokenizeCandidates(this.tokenizer, request.state, question, options, sequenceBudget);
|
| 530 |
const tokenized = performance.now();
|
| 531 |
+
const padded = padCandidates(encoded, slots, liveSequenceLength, this.paddingId);
|
| 532 |
let traceNoise: Float32Array | null = null;
|
| 533 |
const feeds: Record<string, unknown> = {
|
| 534 |
+
input_ids: tensor(this.runtime, "int64", padded.inputIds, [1, slots, liveSequenceLength]),
|
| 535 |
+
attention_mask: tensor(this.runtime, "int64", padded.attentionMask, [1, slots, liveSequenceLength]),
|
| 536 |
+
token_type_ids: tensor(this.runtime, "int64", padded.tokenTypeIds, [1, slots, liveSequenceLength]),
|
| 537 |
option_mask: tensor(this.runtime, "bool", padded.optionMask, [1, slots]),
|
| 538 |
};
|
| 539 |
if (this.backend === "diffusion") {
|
|
|
|
| 582 |
live_candidates: encoded.length,
|
| 583 |
allocated_candidates: slots,
|
| 584 |
live_tokens: encoded.reduce((total, candidate) => total + candidate.inputIds.length, 0),
|
| 585 |
+
sequence_length: liveSequenceLength,
|
| 586 |
tokenizeMs: tokenized - tokenizeStarted,
|
| 587 |
prepareFeedMs: inferenceStarted - tokenized,
|
| 588 |
inferenceAndReadbackMs: readbackCompleted - inferenceStarted,
|
sdk/typescript/test/extension.test.mjs
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import test from "node:test";
|
| 2 |
+
import assert from "node:assert/strict";
|
| 3 |
+
import { readFile } from "node:fs/promises";
|
| 4 |
+
import { requiredFiles, safePath, digest, verifyBundle, makeRequest } from "../extension/bundle.ts";
|
| 5 |
+
|
| 6 |
+
async function fixture() {
|
| 7 |
+
const asset = new TextEncoder().encode("fixture only").buffer;
|
| 8 |
+
const hash = await digest(asset);
|
| 9 |
+
const files = [
|
| 10 |
+
{ path: "graph.onnx", role: "model_graph" },
|
| 11 |
+
{ path: "tokenizer/tokenizer.json", role: "tokenizer" },
|
| 12 |
+
{ path: "tokenizer/tokenizer_config.json", role: "tokenizer" },
|
| 13 |
+
].map((file) => ({ ...file, bytes: asset.byteLength, sha256: hash }));
|
| 14 |
+
const manifestBytes = new TextEncoder().encode(JSON.stringify({ schema_version: "vons.bundle.manifest/v1", metadata: { backend: "direct" }, files })).buffer;
|
| 15 |
+
return { manifestPath: "bundle-manifest-v1.json", manifestBytes, manifestHash: await digest(manifestBytes), backend: "direct", modelId: "test", assets: files.map((file) => ({ path: file.path, bytes: asset })) };
|
| 16 |
+
}
|
| 17 |
+
test("extension verifies exact manifest and asset bytes, including retained bundles", async () => {
|
| 18 |
+
const bundle = await fixture(); await verifyBundle(bundle);
|
| 19 |
+
await assert.rejects(verifyBundle({ ...bundle, manifestHash: "0".repeat(64) }), /integrity/);
|
| 20 |
+
await assert.rejects(verifyBundle({ ...bundle, assets: bundle.assets.slice(1) }), /contents/);
|
| 21 |
+
const corrupt = structuredClone(bundle); new Uint8Array(corrupt.assets[0].bytes)[0] = 0;
|
| 22 |
+
await assert.rejects(verifyBundle(corrupt), /SHA-256/);
|
| 23 |
+
});
|
| 24 |
+
test("extension rejects escaped paths and incomplete model contracts", async () => {
|
| 25 |
+
for (const value of ["../model", "https://evil.test/x", "%2e%2e/x", "a\\b", "/abs", "a?token", "a//b", "x\u0000y"]) assert.equal(safePath(value), false);
|
| 26 |
+
assert.equal(safePath("tokenizer/tokenizer.json"), true);
|
| 27 |
+
const bundle = await fixture(); const manifest = JSON.parse(new TextDecoder().decode(bundle.manifestBytes));
|
| 28 |
+
manifest.metadata.graph = { external_data_locations: ["unknown.onnx.data"] };
|
| 29 |
+
assert.throws(() => requiredFiles(new TextEncoder().encode(JSON.stringify(manifest)).buffer), /external-data/);
|
| 30 |
+
});
|
| 31 |
+
test("extension accepts bounded unique candidates and refuses empty or duplicate input", () => {
|
| 32 |
+
assert.deepEqual(makeRequest(" ready ", "Choose", "A\r\nB\n").questions[0].options, ["A", "B"]);
|
| 33 |
+
assert.throws(() => makeRequest("", "Choose", "A\nB"), /empty/);
|
| 34 |
+
assert.throws(() => makeRequest("ready", "Choose", "A\nA"), /unique/);
|
| 35 |
+
assert.throws(() => makeRequest("ready", "Choose", "A"), /2..32/);
|
| 36 |
+
});
|
| 37 |
+
test("extension has no page access, remote execution permission, or automatic action script", async () => {
|
| 38 |
+
const manifest = JSON.parse(await readFile(new URL("../extension/manifest.json", import.meta.url), "utf8"));
|
| 39 |
+
assert.equal(manifest.manifest_version, 3);
|
| 40 |
+
assert.deepEqual(manifest.permissions, ["sidePanel"]);
|
| 41 |
+
assert.equal(manifest.host_permissions, undefined);
|
| 42 |
+
assert.equal(manifest.content_scripts, undefined);
|
| 43 |
+
assert.equal(manifest.externally_connectable, undefined);
|
| 44 |
+
assert.match(manifest.content_security_policy.extension_pages, /connect-src 'self';/);
|
| 45 |
+
assert.doesNotMatch(manifest.content_security_policy.extension_pages, /'unsafe-eval'|https:|http:|\*/);
|
| 46 |
+
});
|
sdk/typescript/test/onnx.test.mjs
CHANGED
|
@@ -51,7 +51,8 @@ test("tokenizer ids, masks, and overflow behavior are explicit", async () => {
|
|
| 51 |
{ tokenizer_class: "BertTokenizer", unk_token: "[UNK]", cls_token: "[CLS]", sep_token: "[SEP]", model_max_length: 512 },
|
| 52 |
);
|
| 53 |
const question = { id: "q", type: "choice", prompt: "Pick", options: ["A/B?", "中文"] };
|
| 54 |
-
const values = onnx.tokenizeCandidates(tokenizer, "ready", question, question.options, 512);
|
|
|
|
| 55 |
assert.deepEqual(values[0].inputIds, [1, 3, 4, 7, 5, 6, 7, 8, 10, 9, 11, 2]);
|
| 56 |
assert.deepEqual(values[1].inputIds, [1, 3, 4, 7, 5, 6, 7, 12, 13, 2]);
|
| 57 |
assert.equal(values[0].inputIds.length, values[0].attentionMask.length);
|
|
@@ -59,6 +60,60 @@ test("tokenizer ids, masks, and overflow behavior are explicit", async () => {
|
|
| 59 |
assert.throws(() => onnx.tokenizeCandidates(tokenizer, "x", question, ["x".repeat(10000)], 4), /bundle limit/);
|
| 60 |
});
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
test("manifest integrity is checked before a runtime session is created", async () => {
|
| 63 |
const manifest = testManifest();
|
| 64 |
const original = manifest.files.find((item) => item.role === "model_graph");
|
|
|
|
| 51 |
{ tokenizer_class: "BertTokenizer", unk_token: "[UNK]", cls_token: "[CLS]", sep_token: "[SEP]", model_max_length: 512 },
|
| 52 |
);
|
| 53 |
const question = { id: "q", type: "choice", prompt: "Pick", options: ["A/B?", "中文"] };
|
| 54 |
+
const { candidates: values, liveSequenceLength } = onnx.tokenizeCandidates(tokenizer, "ready", question, question.options, 512);
|
| 55 |
+
assert.equal(liveSequenceLength, Math.max(values[0].inputIds.length, values[1].inputIds.length));
|
| 56 |
assert.deepEqual(values[0].inputIds, [1, 3, 4, 7, 5, 6, 7, 8, 10, 9, 11, 2]);
|
| 57 |
assert.deepEqual(values[1].inputIds, [1, 3, 4, 7, 5, 6, 7, 12, 13, 2]);
|
| 58 |
assert.equal(values[0].inputIds.length, values[0].attentionMask.length);
|
|
|
|
| 60 |
assert.throws(() => onnx.tokenizeCandidates(tokenizer, "x", question, ["x".repeat(10000)], 4), /bundle limit/);
|
| 61 |
});
|
| 62 |
|
| 63 |
+
test("tokenizeCandidates returns the longest live tokenized candidate length, bounded at 512", async () => {
|
| 64 |
+
const { Tokenizer } = await import("@huggingface/tokenizers");
|
| 65 |
+
const tokenizer = new Tokenizer(
|
| 66 |
+
{
|
| 67 |
+
version: "1.0",
|
| 68 |
+
added_tokens: [],
|
| 69 |
+
normalizer: { type: "BertNormalizer", clean_text: true, handle_chinese_chars: true, strip_accents: null, lowercase: true },
|
| 70 |
+
pre_tokenizer: { type: "BertPreTokenizer" },
|
| 71 |
+
post_processor: { type: "BertProcessing", sep: ["[SEP]", 2], cls: ["<[BOS_never_used_51bce0c785ca2f68081bfa7d91973934]>", 1] },
|
| 72 |
+
decoder: { type: "WordPiece", prefix: "##", cleanup: true },
|
| 73 |
+
model: { type: "WordPiece", unk_token: "[UNK]", continuing_subword_prefix: "##", max_input_chars_per_word: 100,
|
| 74 |
+
vocab: Object.assign({ "[UNK]": 0, "[CLS]": 1, "[SEP]": 2, "question": 101, "candidate": 102, ":": 103 },
|
| 75 |
+
Object.fromEntries(Array.from({ length: 50 }, (_, i) => [`t${i}`, i + 3]))) },
|
| 76 |
+
},
|
| 77 |
+
{ tokenizer_class: "BertTokenizer", unk_token: "[UNK]", cls_token: "[CLS]", sep_token: "[SEP]", model_max_length: 512 },
|
| 78 |
+
);
|
| 79 |
+
const shortQuestion = { id: "q", type: "choice", prompt: "Pick", options: ["t0 t1", "t0 t1 t2 t3"] };
|
| 80 |
+
const short = onnx.tokenizeCandidates(tokenizer, "", shortQuestion, shortQuestion.options, 512);
|
| 81 |
+
assert.ok(short.liveSequenceLength >= 1);
|
| 82 |
+
assert.equal(short.liveSequenceLength, Math.max(short.candidates[0].inputIds.length, short.candidates[1].inputIds.length));
|
| 83 |
+
assert.ok(short.liveSequenceLength <= 512);
|
| 84 |
+
|
| 85 |
+
const capped = onnx.tokenizeCandidates(tokenizer, "", shortQuestion, shortQuestion.options, 64);
|
| 86 |
+
assert.equal(capped.liveSequenceLength, short.liveSequenceLength);
|
| 87 |
+
|
| 88 |
+
assert.throws(() => onnx.tokenizeCandidates(tokenizer, "", shortQuestion, shortQuestion.options, 0), /sequenceBudget/);
|
| 89 |
+
assert.throws(() => onnx.tokenizeCandidates(tokenizer, "", shortQuestion, shortQuestion.options, 513), /sequenceBudget/);
|
| 90 |
+
});
|
| 91 |
+
|
| 92 |
+
test("variable sequence length regression: widths 64 and 128 produce identical live-token prefixes", async () => {
|
| 93 |
+
const { Tokenizer } = await import("@huggingface/tokenizers");
|
| 94 |
+
const tokenizer = new Tokenizer(
|
| 95 |
+
{
|
| 96 |
+
version: "1.0",
|
| 97 |
+
added_tokens: [],
|
| 98 |
+
normalizer: { type: "BertNormalizer", clean_text: true, handle_chinese_chars: true, strip_accents: null, lowercase: true },
|
| 99 |
+
pre_tokenizer: { type: "BertPreTokenizer" },
|
| 100 |
+
post_processor: { type: "BertProcessing", sep: ["[SEP]", 2], cls: ["<[BOS_never_used_51bce0c785ca2f68081bfa7d91973934]>", 1] },
|
| 101 |
+
decoder: { type: "WordPiece", prefix: "##", cleanup: true },
|
| 102 |
+
model: { type: "WordPiece", unk_token: "[UNK]", continuing_subword_prefix: "##", max_input_chars_per_word: 100,
|
| 103 |
+
vocab: { "[UNK]": 0, "[CLS]": 1, "[SEP]": 2, ready: 3, question: 4, pick: 5, candidate: 6, ":": 7, a: 8, b: 9, c: 10, d: 11 } },
|
| 104 |
+
},
|
| 105 |
+
{ tokenizer_class: "BertTokenizer", unk_token: "[UNK]", cls_token: "[CLS]", sep_token: "[SEP]", model_max_length: 512 },
|
| 106 |
+
);
|
| 107 |
+
const question = { id: "q", type: "choice", prompt: "Pick", options: ["a b", "c d"] };
|
| 108 |
+
const result64 = onnx.tokenizeCandidates(tokenizer, "ready", question, question.options, 64);
|
| 109 |
+
const result128 = onnx.tokenizeCandidates(tokenizer, "ready", question, question.options, 128);
|
| 110 |
+
assert.equal(result64.liveSequenceLength, result128.liveSequenceLength);
|
| 111 |
+
for (let index = 0; index < question.options.length; index += 1) {
|
| 112 |
+
assert.deepEqual(result64.candidates[index].inputIds, result128.candidates[index].inputIds);
|
| 113 |
+
assert.deepEqual(result64.candidates[index].attentionMask, result128.candidates[index].attentionMask);
|
| 114 |
+
}
|
| 115 |
+
});
|
| 116 |
+
|
| 117 |
test("manifest integrity is checked before a runtime session is created", async () => {
|
| 118 |
const manifest = testManifest();
|
| 119 |
const original = manifest.files.find((item) => item.role === "model_graph");
|
tests/test_calibration.py
CHANGED
|
@@ -9,11 +9,14 @@ from vons.calibration import (
|
|
| 9 |
ABSTAIN_ALL_THRESHOLD,
|
| 10 |
CALIBRATION_VERSION,
|
| 11 |
apply_calibration,
|
|
|
|
| 12 |
calibrate_report,
|
| 13 |
fit_abstention_threshold,
|
| 14 |
fit_abstention_threshold_details,
|
|
|
|
| 15 |
fit_temperature,
|
| 16 |
fit_temperature_details,
|
|
|
|
| 17 |
)
|
| 18 |
from vons.data import Example, write_jsonl
|
| 19 |
from vons.evaluation import METRIC_VERSION, Prediction, nll, summarize, write_report
|
|
@@ -46,21 +49,97 @@ class CalibrationTests(unittest.TestCase):
|
|
| 46 |
details = fit_temperature_details(examples, predictions)
|
| 47 |
calibrated = apply_calibration(examples, predictions, temperature=details.temperature, abstention_threshold=0.0)
|
| 48 |
|
| 49 |
-
self.
|
|
|
|
| 50 |
self.assertEqual(details.rows, 4)
|
| 51 |
self.assertFalse(details.at_lower_bound)
|
| 52 |
self.assertFalse(details.at_upper_bound)
|
| 53 |
self.assertEqual(fit_temperature(examples, predictions), details.temperature)
|
| 54 |
self.assertLess(nll(examples, calibrated), nll(examples, predictions))
|
| 55 |
|
| 56 |
-
def
|
| 57 |
examples = [_example(f"s{index}", "a", True) for index in range(4)]
|
| 58 |
predictions = [Prediction(f"s{index}", "a", {"a": 0.9, "b": 0.1}, 0.9, False) for index in range(4)]
|
| 59 |
|
| 60 |
details = fit_temperature_details(examples, predictions)
|
| 61 |
|
| 62 |
-
self.
|
| 63 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
|
| 65 |
def test_temperature_scaling_preserves_true_zero_support(self) -> None:
|
| 66 |
examples = [_example("zero", "a", True)]
|
|
@@ -212,15 +291,20 @@ class CalibrationTests(unittest.TestCase):
|
|
| 212 |
"test_report": test_report,
|
| 213 |
}
|
| 214 |
|
| 215 |
-
calibrate_report(**arguments, output=output,
|
| 216 |
payload = json.loads(output.read_text(encoding="utf-8"), parse_constant=_reject_constant)
|
| 217 |
|
| 218 |
fit = payload["calibration"]
|
| 219 |
self.assertEqual(fit["version"], CALIBRATION_VERSION)
|
| 220 |
self.assertEqual(fit["answerability_threshold"], 0.8)
|
| 221 |
self.assertEqual((fit["temperature_fit_rows"], fit["threshold_fit_rows"], fit["rows"]), (2, 3, 3))
|
| 222 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
self.assertLessEqual(fit["nll_after"], fit["nll_before"])
|
|
|
|
| 224 |
self.assertEqual(payload["inputs"]["test_report"]["sha256"], hashlib.sha256(test_report.read_bytes()).hexdigest())
|
| 225 |
self.assertEqual(payload["model_identity"]["model_id"], "vons-test")
|
| 226 |
self.assertTrue(any("checkpoint_sha256 unavailable" in item for item in payload["limitations"]))
|
|
|
|
| 9 |
ABSTAIN_ALL_THRESHOLD,
|
| 10 |
CALIBRATION_VERSION,
|
| 11 |
apply_calibration,
|
| 12 |
+
apply_vector_calibration,
|
| 13 |
calibrate_report,
|
| 14 |
fit_abstention_threshold,
|
| 15 |
fit_abstention_threshold_details,
|
| 16 |
+
fit_brier_abstention_threshold_details,
|
| 17 |
fit_temperature,
|
| 18 |
fit_temperature_details,
|
| 19 |
+
fit_vector_scaling_details,
|
| 20 |
)
|
| 21 |
from vons.data import Example, write_jsonl
|
| 22 |
from vons.evaluation import METRIC_VERSION, Prediction, nll, summarize, write_report
|
|
|
|
| 49 |
details = fit_temperature_details(examples, predictions)
|
| 50 |
calibrated = apply_calibration(examples, predictions, temperature=details.temperature, abstention_threshold=0.0)
|
| 51 |
|
| 52 |
+
self.assertGreater(details.temperature, details.lower)
|
| 53 |
+
self.assertLess(details.temperature, details.upper)
|
| 54 |
self.assertEqual(details.rows, 4)
|
| 55 |
self.assertFalse(details.at_lower_bound)
|
| 56 |
self.assertFalse(details.at_upper_bound)
|
| 57 |
self.assertEqual(fit_temperature(examples, predictions), details.temperature)
|
| 58 |
self.assertLess(nll(examples, calibrated), nll(examples, predictions))
|
| 59 |
|
| 60 |
+
def test_temperature_regularization_prevents_boundary_locking(self) -> None:
|
| 61 |
examples = [_example(f"s{index}", "a", True) for index in range(4)]
|
| 62 |
predictions = [Prediction(f"s{index}", "a", {"a": 0.9, "b": 0.1}, 0.9, False) for index in range(4)]
|
| 63 |
|
| 64 |
details = fit_temperature_details(examples, predictions)
|
| 65 |
|
| 66 |
+
self.assertFalse(details.at_lower_bound)
|
| 67 |
+
self.assertFalse(details.at_upper_bound)
|
| 68 |
+
self.assertGreater(details.temperature, 0.1)
|
| 69 |
+
self.assertLess(details.temperature, 10.0)
|
| 70 |
+
|
| 71 |
+
unregularized = fit_temperature_details(
|
| 72 |
+
examples, predictions, log_temperature_l2=0.0
|
| 73 |
+
)
|
| 74 |
+
self.assertTrue(unregularized.at_lower_bound)
|
| 75 |
+
self.assertAlmostEqual(unregularized.temperature, 0.1)
|
| 76 |
+
|
| 77 |
+
def test_vector_scaling_fits_all_classes_inside_bounds(self) -> None:
|
| 78 |
+
options = ("a", "b", "c", "d")
|
| 79 |
+
examples = []
|
| 80 |
+
predictions = []
|
| 81 |
+
for index, label in enumerate(options * 5):
|
| 82 |
+
examples.append(
|
| 83 |
+
Example(
|
| 84 |
+
str(index),
|
| 85 |
+
"calibration",
|
| 86 |
+
"state",
|
| 87 |
+
"choose",
|
| 88 |
+
options,
|
| 89 |
+
label,
|
| 90 |
+
True,
|
| 91 |
+
"calibration",
|
| 92 |
+
{},
|
| 93 |
+
{"scenario_id": str(index)},
|
| 94 |
+
)
|
| 95 |
+
)
|
| 96 |
+
probabilities = {option: 0.1 for option in options}
|
| 97 |
+
probabilities[label] = 0.7
|
| 98 |
+
predictions.append(
|
| 99 |
+
Prediction(str(index), label, probabilities, 0.7, False)
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
details = fit_vector_scaling_details(examples, predictions)
|
| 103 |
+
calibrated = apply_vector_calibration(
|
| 104 |
+
examples,
|
| 105 |
+
predictions,
|
| 106 |
+
temperatures=details.temperatures,
|
| 107 |
+
biases=details.biases,
|
| 108 |
+
abstention_threshold=0.0,
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
self.assertEqual(set(details.temperatures), set(options))
|
| 112 |
+
self.assertTrue(all(0.1 < value < 10.0 for value in details.temperatures.values()))
|
| 113 |
+
self.assertFalse(details.at_lower_bound)
|
| 114 |
+
self.assertFalse(details.at_upper_bound)
|
| 115 |
+
self.assertLess(details.nll_after, details.nll_before)
|
| 116 |
+
self.assertLess(nll(examples, calibrated), nll(examples, predictions))
|
| 117 |
+
|
| 118 |
+
def test_brier_threshold_is_fitted_from_calibrated_losses(self) -> None:
|
| 119 |
+
examples = [
|
| 120 |
+
_example("strong", "a", True),
|
| 121 |
+
_example("weak", "b", True),
|
| 122 |
+
_example("wrong", "b", True),
|
| 123 |
+
]
|
| 124 |
+
predictions = [
|
| 125 |
+
Prediction("strong", "a", {"a": 0.95, "b": 0.05}, 0.95, False),
|
| 126 |
+
Prediction("weak", "b", {"a": 0.4, "b": 0.6}, 0.6, False),
|
| 127 |
+
Prediction("wrong", "a", {"a": 0.9, "b": 0.1}, 0.9, False),
|
| 128 |
+
]
|
| 129 |
+
parameters = {"a": 1.0, "b": 1.0}
|
| 130 |
+
|
| 131 |
+
details = fit_brier_abstention_threshold_details(
|
| 132 |
+
examples,
|
| 133 |
+
predictions,
|
| 134 |
+
temperatures=parameters,
|
| 135 |
+
biases={"a": 0.0, "b": 0.0},
|
| 136 |
+
target_brier=0.25,
|
| 137 |
+
)
|
| 138 |
+
|
| 139 |
+
self.assertFalse(details.abstain_all)
|
| 140 |
+
self.assertEqual(details.target_brier, 0.25)
|
| 141 |
+
self.assertLessEqual(details.covered_brier, details.target_brier)
|
| 142 |
+
self.assertGreater(details.threshold, 0.0)
|
| 143 |
|
| 144 |
def test_temperature_scaling_preserves_true_zero_support(self) -> None:
|
| 145 |
examples = [_example("zero", "a", True)]
|
|
|
|
| 291 |
"test_report": test_report,
|
| 292 |
}
|
| 293 |
|
| 294 |
+
calibrate_report(**arguments, output=output, target_brier=0.2)
|
| 295 |
payload = json.loads(output.read_text(encoding="utf-8"), parse_constant=_reject_constant)
|
| 296 |
|
| 297 |
fit = payload["calibration"]
|
| 298 |
self.assertEqual(fit["version"], CALIBRATION_VERSION)
|
| 299 |
self.assertEqual(fit["answerability_threshold"], 0.8)
|
| 300 |
self.assertEqual((fit["temperature_fit_rows"], fit["threshold_fit_rows"], fit["rows"]), (2, 3, 3))
|
| 301 |
+
self.assertEqual(fit["calibration_method"], "vector_scaling")
|
| 302 |
+
self.assertEqual(set(fit["temperatures"]), {"a", "b"})
|
| 303 |
+
self.assertEqual(set(fit["biases"]), {"a", "b"})
|
| 304 |
+
self.assertFalse(fit["temperature_at_lower_bound"])
|
| 305 |
+
self.assertEqual(fit["target_brier"], 0.2)
|
| 306 |
self.assertLessEqual(fit["nll_after"], fit["nll_before"])
|
| 307 |
+
self.assertLessEqual(fit["brier_after"], fit["brier_before"])
|
| 308 |
self.assertEqual(payload["inputs"]["test_report"]["sha256"], hashlib.sha256(test_report.read_bytes()).hexdigest())
|
| 309 |
self.assertEqual(payload["model_identity"]["model_id"], "vons-test")
|
| 310 |
self.assertTrue(any("checkpoint_sha256 unavailable" in item for item in payload["limitations"]))
|
tests/test_data.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
import tempfile
|
| 2 |
import unittest
|
|
|
|
| 3 |
from pathlib import Path
|
| 4 |
|
| 5 |
from vons.data import dataset_manifest, read_jsonl, smoke_examples, synthetic_examples, write_jsonl
|
|
@@ -17,7 +18,25 @@ class DataTests(unittest.TestCase):
|
|
| 17 |
def test_reproducible_generation(self) -> None:
|
| 18 |
self.assertEqual([row.to_mapping() for row in synthetic_examples(20, seed=9)], [row.to_mapping() for row in synthetic_examples(20, seed=9)])
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
if __name__ == "__main__":
|
| 22 |
unittest.main()
|
| 23 |
-
|
|
|
|
| 1 |
import tempfile
|
| 2 |
import unittest
|
| 3 |
+
from collections import Counter
|
| 4 |
from pathlib import Path
|
| 5 |
|
| 6 |
from vons.data import dataset_manifest, read_jsonl, smoke_examples, synthetic_examples, write_jsonl
|
|
|
|
| 18 |
def test_reproducible_generation(self) -> None:
|
| 19 |
self.assertEqual([row.to_mapping() for row in synthetic_examples(20, seed=9)], [row.to_mapping() for row in synthetic_examples(20, seed=9)])
|
| 20 |
|
| 21 |
+
def test_synthetic_generation_balances_all_actions_and_rules(self) -> None:
|
| 22 |
+
rows = synthetic_examples(200, seed=9)
|
| 23 |
+
answerable = [row for row in rows if row.answerable]
|
| 24 |
+
counts = Counter(row.label for row in answerable)
|
| 25 |
+
|
| 26 |
+
self.assertEqual(set(counts), {"call_tool", "clarify", "refuse", "respond_directly"})
|
| 27 |
+
self.assertLessEqual(max(counts.values()) - min(counts.values()), 1)
|
| 28 |
+
for count in counts.values():
|
| 29 |
+
self.assertAlmostEqual(count / len(answerable), 0.25, delta=0.01)
|
| 30 |
+
self.assertTrue(all(row.provenance.get("rule") for row in rows))
|
| 31 |
+
self.assertGreater(len({row.options.index(row.label) for row in answerable}), 1)
|
| 32 |
+
|
| 33 |
+
def test_manifest_reports_label_and_abstention_counts(self) -> None:
|
| 34 |
+
rows = synthetic_examples(52)
|
| 35 |
+
manifest = dataset_manifest("synthetic.jsonl", rows)
|
| 36 |
+
|
| 37 |
+
self.assertEqual(sum(manifest["labels"].values()) + manifest["unanswerable_rows"], 52)
|
| 38 |
+
self.assertEqual(set(manifest["labels"]), {"call_tool", "clarify", "refuse", "respond_directly"})
|
| 39 |
+
|
| 40 |
|
| 41 |
if __name__ == "__main__":
|
| 42 |
unittest.main()
|
|
|
tests/test_diffusion_model.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from itertools import pairwise
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
torch = pytest.importorskip("torch")
|
| 6 |
+
|
| 7 |
+
from vons.models import ModelConfig, build_model, cosine_beta_schedule
|
| 8 |
+
from vons.training import soft_diffusion_targets
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _model(hidden_size=16):
|
| 12 |
+
return build_model(
|
| 13 |
+
"diffusion",
|
| 14 |
+
option_count=32,
|
| 15 |
+
config=ModelConfig(
|
| 16 |
+
hidden_size=hidden_size,
|
| 17 |
+
latent_size=8,
|
| 18 |
+
diffusion_steps=16,
|
| 19 |
+
inference_steps=4,
|
| 20 |
+
attention_heads=2,
|
| 21 |
+
),
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def test_cosine_schedule_reaches_high_terminal_noise():
|
| 26 |
+
betas = cosine_beta_schedule(16)
|
| 27 |
+
alpha_bar = torch.cumprod(1.0 - betas, dim=0)
|
| 28 |
+
|
| 29 |
+
assert betas.shape == (16,)
|
| 30 |
+
assert torch.all(betas[1:] > betas[:-1])
|
| 31 |
+
assert float(alpha_bar[-1]) < 1e-4
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@pytest.mark.parametrize(("options", "sequence"), [(3, 7), (5, 11)])
|
| 35 |
+
def test_token_conditioned_head_accepts_dynamic_option_and_sequence_axes(options, sequence):
|
| 36 |
+
torch.manual_seed(7)
|
| 37 |
+
model = _model().eval()
|
| 38 |
+
output = model(
|
| 39 |
+
torch.randn(2, options),
|
| 40 |
+
torch.randn(2, options, sequence, 16),
|
| 41 |
+
torch.ones(2, options, sequence, dtype=torch.bool),
|
| 42 |
+
torch.tensor([3, 4]),
|
| 43 |
+
torch.ones(2, options, dtype=torch.bool),
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
assert output["logits"].shape == (2, options)
|
| 47 |
+
assert output["answerability"].shape == (2,)
|
| 48 |
+
assert torch.isfinite(output["logits"]).all()
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def test_soft_targets_are_continuous_normalized_and_masked():
|
| 52 |
+
labels = torch.tensor([0, 2])
|
| 53 |
+
option_mask = torch.tensor([[True, True, True, False], [True, True, True, True]])
|
| 54 |
+
|
| 55 |
+
targets = soft_diffusion_targets(labels, option_mask, smoothing=0.1)
|
| 56 |
+
|
| 57 |
+
torch.testing.assert_close(targets.sum(dim=-1), torch.ones(2))
|
| 58 |
+
assert torch.all((targets > 0) == option_mask)
|
| 59 |
+
torch.testing.assert_close(targets[torch.arange(2), labels], torch.tensor([0.9, 0.9]))
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def test_token_conditioned_soft_target_loss_strictly_decreases():
|
| 63 |
+
torch.manual_seed(7)
|
| 64 |
+
batch, options, sequence, hidden = 8, 4, 6, 16
|
| 65 |
+
model = _model(hidden)
|
| 66 |
+
sequence_hidden = torch.randn(batch, options, sequence, hidden)
|
| 67 |
+
attention_mask = torch.ones(batch, options, sequence, dtype=torch.bool)
|
| 68 |
+
option_mask = torch.ones(batch, options, dtype=torch.bool)
|
| 69 |
+
labels = torch.arange(batch) % options
|
| 70 |
+
targets = soft_diffusion_targets(labels, option_mask, smoothing=0.1)
|
| 71 |
+
noisy_scores = targets * 0.7 + torch.randn_like(targets) * 0.3
|
| 72 |
+
timestep = torch.full((batch,), 5, dtype=torch.long)
|
| 73 |
+
optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
|
| 74 |
+
losses = []
|
| 75 |
+
|
| 76 |
+
for _ in range(12):
|
| 77 |
+
optimizer.zero_grad()
|
| 78 |
+
output = model(
|
| 79 |
+
noisy_scores,
|
| 80 |
+
sequence_hidden,
|
| 81 |
+
attention_mask,
|
| 82 |
+
timestep,
|
| 83 |
+
option_mask,
|
| 84 |
+
)
|
| 85 |
+
loss = torch.nn.functional.cross_entropy(output["logits"], targets)
|
| 86 |
+
loss.backward()
|
| 87 |
+
optimizer.step()
|
| 88 |
+
losses.append(float(loss.detach()))
|
| 89 |
+
|
| 90 |
+
assert all(current < previous for previous, current in pairwise(losses))
|
tests/test_inference.py
ADDED
|
@@ -0,0 +1,323 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Regression tests for runtime inference helpers.
|
| 2 |
+
|
| 3 |
+
Covers the public task-1 contract:
|
| 4 |
+
|
| 5 |
+
* Tensors use the longest live tokenized candidate in the request, bounded at 512.
|
| 6 |
+
* Exported and runtime ONNX metadata preserves dynamic ``sequence_length``
|
| 7 |
+
and dynamic ``live_candidates`` / ``options`` when the contract supports it.
|
| 8 |
+
* Variable-length regression for shapes [1, X, 64] and [1, X, 128] produces
|
| 9 |
+
equivalent candidate scores when padded inputs are identical (i.e. the same
|
| 10 |
+
prefix of tokens in both cases).
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
from dataclasses import dataclass
|
| 16 |
+
|
| 17 |
+
import pytest
|
| 18 |
+
|
| 19 |
+
from vons.contract import Backend
|
| 20 |
+
from vons.inference import (
|
| 21 |
+
MAX_SEQUENCE_BUDGET,
|
| 22 |
+
TokenizedCandidate,
|
| 23 |
+
candidate_text,
|
| 24 |
+
compute_live_sequence_length,
|
| 25 |
+
pad_candidates,
|
| 26 |
+
prepare_question_tensors,
|
| 27 |
+
resolve_allocated_slots,
|
| 28 |
+
serialize_state,
|
| 29 |
+
tokenize_candidates,
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
@dataclass
|
| 34 |
+
class _StubTokenizer:
|
| 35 |
+
"""Word-Piece-like stub with the exact tokens the tests need."""
|
| 36 |
+
|
| 37 |
+
vocab: dict[str, int]
|
| 38 |
+
pad_token_id: int = 0
|
| 39 |
+
cls_token_id: int = 1
|
| 40 |
+
sep_token_id: int = 2
|
| 41 |
+
|
| 42 |
+
def encode(self, text, **_kwargs):
|
| 43 |
+
ids = [self.cls_token_id]
|
| 44 |
+
for chunk in text.split():
|
| 45 |
+
ids.append(self.vocab.get(chunk.lower(), self.vocab.get("[UNK]", 0)))
|
| 46 |
+
ids.append(self.sep_token_id)
|
| 47 |
+
attention_mask = [1] * len(ids)
|
| 48 |
+
token_type_ids = [0] * len(ids)
|
| 49 |
+
|
| 50 |
+
class _Result:
|
| 51 |
+
pass
|
| 52 |
+
|
| 53 |
+
result = _Result()
|
| 54 |
+
result.ids = ids
|
| 55 |
+
result.attention_mask = attention_mask
|
| 56 |
+
result.token_type_ids = token_type_ids
|
| 57 |
+
return result
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _tokenizer():
|
| 61 |
+
vocab = {
|
| 62 |
+
"[UNK]": 0,
|
| 63 |
+
"ready": 3,
|
| 64 |
+
"question": 4,
|
| 65 |
+
"pick": 5,
|
| 66 |
+
"candidate": 6,
|
| 67 |
+
":": 7,
|
| 68 |
+
"a": 8,
|
| 69 |
+
"b": 9,
|
| 70 |
+
"/": 10,
|
| 71 |
+
"?": 11,
|
| 72 |
+
"中": 12,
|
| 73 |
+
"文": 13,
|
| 74 |
+
"long": 14,
|
| 75 |
+
"option": 15,
|
| 76 |
+
"label": 16,
|
| 77 |
+
"three": 17,
|
| 78 |
+
"words": 18,
|
| 79 |
+
"here": 19,
|
| 80 |
+
"yes": 20,
|
| 81 |
+
"no": 21,
|
| 82 |
+
"{": 22,
|
| 83 |
+
"}": 23,
|
| 84 |
+
'"z"': 24,
|
| 85 |
+
"1": 25,
|
| 86 |
+
'"a"': 26,
|
| 87 |
+
'"x"': 27,
|
| 88 |
+
"2": 28,
|
| 89 |
+
"question:": 4,
|
| 90 |
+
"candidate:": 6,
|
| 91 |
+
}
|
| 92 |
+
return _StubTokenizer(vocab)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
class _ObjectQuestion:
|
| 96 |
+
def __init__(self, qid: str, qtype: str, prompt: str, options: tuple[str, ...]):
|
| 97 |
+
self.id = qid
|
| 98 |
+
self.type = qtype
|
| 99 |
+
self.prompt = prompt
|
| 100 |
+
self.options = options
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
class _ObjectRequest:
|
| 104 |
+
def __init__(self, state, questions: list[_ObjectQuestion], backend=Backend.DIRECT, seed=None):
|
| 105 |
+
self.state = state
|
| 106 |
+
self.questions = tuple(questions)
|
| 107 |
+
self.backend = backend
|
| 108 |
+
self.seed = seed
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _question(qid="q", prompt="Pick", options=("A", "B"), qtype="choice"):
|
| 112 |
+
return _ObjectQuestion(qid, qtype, prompt, tuple(options))
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _request(state="ready", questions=None, backend=Backend.DIRECT, seed=None):
|
| 116 |
+
return _ObjectRequest(state, questions or [_question()], backend, seed)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def test_serialize_state_sorts_object_keys_for_determinism() -> None:
|
| 120 |
+
assert serialize_state({"z": 1, "a": ["x", 2]}) == '{"a":["x",2],"z":1}'
|
| 121 |
+
assert serialize_state("already a string") == "already a string"
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def test_candidate_text_template_matches_web_runtime() -> None:
|
| 125 |
+
question = _question(prompt="Pick", options=("A",))
|
| 126 |
+
assert (
|
| 127 |
+
candidate_text({"z": 1, "a": ["x", 2]}, question, "A")
|
| 128 |
+
== '{"a":["x",2],"z":1}\nQuestion: Pick\nCandidate: A'
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def test_tokenize_candidates_rejects_over_budget_aggregate(tmp_path) -> None:
|
| 133 |
+
tokenizer = _tokenizer()
|
| 134 |
+
question = _question(options=("A/B?", "中文"))
|
| 135 |
+
with pytest.raises(ValueError, match="aggregate input budget"):
|
| 136 |
+
tokenize_candidates(tokenizer, "ready", question, question.options, max_sequence_budget=3)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def test_compute_live_sequence_length_picks_longest_live_candidate() -> None:
|
| 140 |
+
short = TokenizedCandidate((1, 2), (1, 1), (0, 0))
|
| 141 |
+
medium = TokenizedCandidate((1, 3, 2), (1, 1, 1), (0, 0, 0))
|
| 142 |
+
long = TokenizedCandidate((1, 3, 4, 2), (1, 1, 1, 1), (0, 0, 0, 0))
|
| 143 |
+
assert compute_live_sequence_length([short, medium, long]) == 4
|
| 144 |
+
assert compute_live_sequence_length([short, medium]) == 3
|
| 145 |
+
assert compute_live_sequence_length([short]) == 2
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def test_compute_live_sequence_length_clamps_to_manifest_budget() -> None:
|
| 149 |
+
many = TokenizedCandidate(tuple(range(512)), (1,) * 512, (0,) * 512)
|
| 150 |
+
assert compute_live_sequence_length([many]) == 512
|
| 151 |
+
assert compute_live_sequence_length([many], max_sequence_budget=64) == 64
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def test_compute_live_sequence_length_rejects_empty_candidates() -> None:
|
| 155 |
+
with pytest.raises(ValueError, match="at least one tokenized candidate"):
|
| 156 |
+
compute_live_sequence_length([])
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def test_pad_candidates_uses_exact_live_sequence_length() -> None:
|
| 160 |
+
tokenized = [
|
| 161 |
+
TokenizedCandidate((1, 8, 2), (1, 1, 1), (0, 0, 0)),
|
| 162 |
+
TokenizedCandidate((1, 9, 2), (1, 1, 1), (0, 0, 0)),
|
| 163 |
+
]
|
| 164 |
+
input_ids, attention, token_types, option_mask = pad_candidates(
|
| 165 |
+
tokenized,
|
| 166 |
+
slots=4,
|
| 167 |
+
live_sequence_length=3,
|
| 168 |
+
padding_id=0,
|
| 169 |
+
)
|
| 170 |
+
assert [len(row) for row in input_ids] == [3, 3, 3, 3]
|
| 171 |
+
assert option_mask == [1, 1, 0, 0]
|
| 172 |
+
assert input_ids[0][:3] == [1, 8, 2]
|
| 173 |
+
assert attention[2] == [0, 0, 0]
|
| 174 |
+
assert token_types[1] == [0, 0, 0]
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def test_pad_candidates_rejects_fewer_slots_than_live_candidates() -> None:
|
| 178 |
+
tokenized = [
|
| 179 |
+
TokenizedCandidate((1, 2), (1, 1), (0, 0)),
|
| 180 |
+
TokenizedCandidate((1, 2), (1, 1), (0, 0)),
|
| 181 |
+
]
|
| 182 |
+
with pytest.raises(ValueError, match="slots must be at least"):
|
| 183 |
+
pad_candidates(tokenized, slots=1, live_sequence_length=2, padding_id=0)
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def test_resolve_allocated_slots_diffusion_always_uses_option_count() -> None:
|
| 187 |
+
assert resolve_allocated_slots(Backend.DIFFUSION, 2, option_count=32) == 32
|
| 188 |
+
assert resolve_allocated_slots(Backend.DIFFUSION, 16, option_count=32) == 32
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def test_resolve_allocated_slots_direct_defaults_to_live_count() -> None:
|
| 192 |
+
assert resolve_allocated_slots(Backend.DIRECT, 3, option_count=32) == 3
|
| 193 |
+
assert resolve_allocated_slots(Backend.DIRECT, 12, option_count=32) == 12
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def test_resolve_allocated_slots_direct_honours_explicit_slot_override() -> None:
|
| 197 |
+
assert (
|
| 198 |
+
resolve_allocated_slots(
|
| 199 |
+
Backend.DIRECT,
|
| 200 |
+
2,
|
| 201 |
+
option_count=32,
|
| 202 |
+
direct_option_slots=8,
|
| 203 |
+
)
|
| 204 |
+
== 8
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def test_resolve_allocated_slots_direct_rejects_invalid_slot_override() -> None:
|
| 209 |
+
with pytest.raises(ValueError, match="direct_option_slots must be in"):
|
| 210 |
+
resolve_allocated_slots(
|
| 211 |
+
Backend.DIRECT,
|
| 212 |
+
4,
|
| 213 |
+
option_count=8,
|
| 214 |
+
direct_option_slots=16,
|
| 215 |
+
)
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def test_prepare_question_tensors_uses_dynamic_live_length(tmp_path) -> None:
|
| 219 |
+
tokenizer = _tokenizer()
|
| 220 |
+
request = _request(state="ready", questions=[_question(options=("yes", "no"))])
|
| 221 |
+
padded, raw = prepare_question_tensors(
|
| 222 |
+
tokenizer,
|
| 223 |
+
request,
|
| 224 |
+
request.questions[0],
|
| 225 |
+
option_count=32,
|
| 226 |
+
)
|
| 227 |
+
assert padded.live_sequence_length == max(item.token_count for item in raw)
|
| 228 |
+
assert padded.live_sequence_length <= padded.max_sequence_budget
|
| 229 |
+
for row in padded.input_ids:
|
| 230 |
+
assert len(row) == padded.live_sequence_length
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def test_variable_length_regression_64_and_128_agree_on_identical_prefix() -> None:
|
| 234 |
+
"""Padded tensors of width 64 and 128 with identical tokens are equal.
|
| 235 |
+
|
| 236 |
+
The exported heads are pure feed-forward networks that only attend to
|
| 237 |
+
live tokens via attention_mask. Feeding identical live tokens at two
|
| 238 |
+
different sequence widths must yield the same probability ordering when
|
| 239 |
+
the unpadded content is the same.
|
| 240 |
+
"""
|
| 241 |
+
|
| 242 |
+
def _prefix(width: int):
|
| 243 |
+
ids = [1, 8, 9, 2] + [0] * max(0, width - 4)
|
| 244 |
+
mask = [1, 1, 1, 1] + [0] * max(0, width - 4)
|
| 245 |
+
types = [0] * width
|
| 246 |
+
return TokenizedCandidate(tuple(ids), tuple(mask), tuple(types))
|
| 247 |
+
|
| 248 |
+
tokenized_64 = [_prefix(64) for _ in range(3)]
|
| 249 |
+
tokenized_128 = [_prefix(128) for _ in range(3)]
|
| 250 |
+
padded_64 = pad_candidates(tokenized_64, slots=8, live_sequence_length=64, padding_id=0)
|
| 251 |
+
padded_128 = pad_candidates(tokenized_128, slots=8, live_sequence_length=128, padding_id=0)
|
| 252 |
+
|
| 253 |
+
for option_index in range(3):
|
| 254 |
+
first_64 = padded_64[0][option_index][:4]
|
| 255 |
+
first_128 = padded_128[0][option_index][:4]
|
| 256 |
+
assert first_64 == first_128, "live token content must match"
|
| 257 |
+
assert padded_64[1][option_index][:4] == [1, 1, 1, 1]
|
| 258 |
+
assert padded_128[1][option_index][:4] == [1, 1, 1, 1]
|
| 259 |
+
|
| 260 |
+
assert len(padded_64[0][0]) == 64
|
| 261 |
+
assert len(padded_128[0][0]) == 128
|
| 262 |
+
assert padded_64[3] == [1, 1, 1, 0, 0, 0, 0, 0]
|
| 263 |
+
assert padded_128[3] == [1, 1, 1, 0, 0, 0, 0, 0]
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def test_variable_length_regression_long_candidate_drives_width() -> None:
|
| 267 |
+
tokenizer = _tokenizer()
|
| 268 |
+
short_options = ("A", "B")
|
| 269 |
+
long_options = (
|
| 270 |
+
"long option label here three words",
|
| 271 |
+
"long option label here three words plus padding",
|
| 272 |
+
)
|
| 273 |
+
short_question = _question(options=short_options)
|
| 274 |
+
long_question = _question(options=long_options)
|
| 275 |
+
short, _ = prepare_question_tensors(
|
| 276 |
+
tokenizer,
|
| 277 |
+
_request(questions=[short_question]),
|
| 278 |
+
short_question,
|
| 279 |
+
option_count=32,
|
| 280 |
+
)
|
| 281 |
+
long, _ = prepare_question_tensors(
|
| 282 |
+
tokenizer,
|
| 283 |
+
_request(questions=[long_question]),
|
| 284 |
+
long_question,
|
| 285 |
+
option_count=32,
|
| 286 |
+
)
|
| 287 |
+
assert 1 <= short.live_sequence_length < long.live_sequence_length
|
| 288 |
+
assert long.live_sequence_length <= 512
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def test_max_sequence_budget_is_512_and_clamps() -> None:
|
| 292 |
+
assert MAX_SEQUENCE_BUDGET == 512
|
| 293 |
+
very_long = tuple(range(512))
|
| 294 |
+
many = [
|
| 295 |
+
TokenizedCandidate(very_long, (1,) * 512, (0,) * 512),
|
| 296 |
+
TokenizedCandidate((1, 2), (1, 1), (0, 0)),
|
| 297 |
+
]
|
| 298 |
+
assert compute_live_sequence_length(many) == 512
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
def test_prepare_question_tensors_rejects_over_budget() -> None:
|
| 302 |
+
tokenizer = _tokenizer()
|
| 303 |
+
options = ("A",) * 33
|
| 304 |
+
question = _question(options=options)
|
| 305 |
+
with pytest.raises(ValueError, match="unsupported candidate count"):
|
| 306 |
+
prepare_question_tensors(
|
| 307 |
+
tokenizer,
|
| 308 |
+
_request(questions=[question]),
|
| 309 |
+
question,
|
| 310 |
+
option_count=32,
|
| 311 |
+
)
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def test_pad_candidates_preserves_padding_id_outside_live_tokens() -> None:
|
| 315 |
+
tokenized = [TokenizedCandidate((1, 5, 2), (1, 1, 1), (0, 0, 0))]
|
| 316 |
+
ids, _att, _ttypes, _omask = pad_candidates(
|
| 317 |
+
tokenized,
|
| 318 |
+
slots=2,
|
| 319 |
+
live_sequence_length=5,
|
| 320 |
+
padding_id=42,
|
| 321 |
+
)
|
| 322 |
+
assert ids[1] == [42, 42, 42, 42, 42]
|
| 323 |
+
assert ids[0][3:5] == [42, 42]
|
tests/test_mind2web_adapter.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
|
| 4 |
+
from vons.mind2web_adapter import compress_request, strip_dom_presentation
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass
|
| 8 |
+
class Encoding:
|
| 9 |
+
ids: list[int]
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class WordTokenizer:
|
| 13 |
+
def __init__(self) -> None:
|
| 14 |
+
self.tokens: dict[str, int] = {}
|
| 15 |
+
self.reverse: dict[int, str] = {}
|
| 16 |
+
|
| 17 |
+
def encode(self, text: str) -> Encoding:
|
| 18 |
+
ids = []
|
| 19 |
+
for token in text.split():
|
| 20 |
+
if token not in self.tokens:
|
| 21 |
+
index = len(self.tokens) + 1
|
| 22 |
+
self.tokens[token] = index
|
| 23 |
+
self.reverse[index] = token
|
| 24 |
+
ids.append(self.tokens[token])
|
| 25 |
+
return Encoding(ids)
|
| 26 |
+
|
| 27 |
+
def decode(self, ids: list[int], skip_special_tokens: bool = True) -> str:
|
| 28 |
+
del skip_special_tokens
|
| 29 |
+
return " ".join(self.reverse[index] for index in ids)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def test_dom_compressor_keeps_semantics_and_removes_presentation() -> None:
|
| 33 |
+
raw = (
|
| 34 |
+
'button class="large primary" style="color:red" id=save '
|
| 35 |
+
'type=submit aria-label="Save" data-tracking=secret Save changes '
|
| 36 |
+
"<svg class=icon><path id=shape /></svg>"
|
| 37 |
+
)
|
| 38 |
+
compressed = strip_dom_presentation(raw)
|
| 39 |
+
assert compressed.startswith("button")
|
| 40 |
+
assert "id=save" in compressed
|
| 41 |
+
assert "type=submit" in compressed
|
| 42 |
+
assert 'aria-label="Save"' in compressed
|
| 43 |
+
assert "Save changes" in compressed
|
| 44 |
+
assert "class=" not in compressed
|
| 45 |
+
assert "style=" not in compressed
|
| 46 |
+
assert "data-tracking=" not in compressed
|
| 47 |
+
assert "svg" not in compressed.lower()
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def test_compress_request_enforces_history_candidate_and_total_budgets() -> None:
|
| 51 |
+
tokenizer = WordTokenizer()
|
| 52 |
+
request = {
|
| 53 |
+
"state": json.dumps(
|
| 54 |
+
{
|
| 55 |
+
"goal": " ".join(f"goal-{index}" for index in range(80)),
|
| 56 |
+
"previous_actions": [f"action-{index}" for index in range(5)],
|
| 57 |
+
}
|
| 58 |
+
),
|
| 59 |
+
"question": "choose the next element",
|
| 60 |
+
"candidates": [
|
| 61 |
+
{
|
| 62 |
+
"id": str(index),
|
| 63 |
+
"text": "button class=decorative id=target "
|
| 64 |
+
+ " ".join(f"visible-{part}" for part in range(20)),
|
| 65 |
+
}
|
| 66 |
+
for index in range(8)
|
| 67 |
+
],
|
| 68 |
+
}
|
| 69 |
+
compressed, stats = compress_request(
|
| 70 |
+
request,
|
| 71 |
+
tokenizer,
|
| 72 |
+
max_prompt_tokens=40,
|
| 73 |
+
max_candidate_tokens=3,
|
| 74 |
+
)
|
| 75 |
+
state = json.loads(compressed["state"])
|
| 76 |
+
assert len(state["previous_actions"]) <= 2
|
| 77 |
+
assert stats.history_actions_before == 5
|
| 78 |
+
assert stats.history_actions_after <= 2
|
| 79 |
+
assert stats.prompt_tokens <= 40
|
| 80 |
+
assert stats.max_candidate_tokens <= 3
|
| 81 |
+
assert all(
|
| 82 |
+
len(tokenizer.encode(candidate["text"]).ids) <= 3
|
| 83 |
+
for candidate in compressed["candidates"]
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def test_compress_request_preserves_last_two_actions_when_they_fit() -> None:
|
| 88 |
+
tokenizer = WordTokenizer()
|
| 89 |
+
request = {
|
| 90 |
+
"state": {
|
| 91 |
+
"goal": "save the form",
|
| 92 |
+
"previous_actions": ["first", "second", "third", "fourth"],
|
| 93 |
+
},
|
| 94 |
+
"question": "choose",
|
| 95 |
+
"candidates": [{"id": "save", "text": "button id=save Save"}],
|
| 96 |
+
}
|
| 97 |
+
compressed, stats = compress_request(request, tokenizer)
|
| 98 |
+
state = json.loads(compressed["state"])
|
| 99 |
+
assert state["previous_actions"] == ["third", "fourth"]
|
| 100 |
+
assert stats.history_actions_after == 2
|
| 101 |
+
assert stats.prompt_tokens <= 512
|
tests/test_mind2web_prediction.py
CHANGED
|
@@ -35,6 +35,7 @@ def test_exact_k_and_label_free_contract():
|
|
| 35 |
def test_decode_abstention_finite_and_ids():
|
| 36 |
result = module.decode([0, 10], 10, ["first", "second"])
|
| 37 |
assert result["selection"] == "second"
|
|
|
|
| 38 |
assert sum(result["probabilities"].values()) == pytest.approx(1)
|
| 39 |
assert module.decode([0, 10], -10, ["first", "second"])["selection"] is None
|
| 40 |
with pytest.raises(ValueError, match="invalid scores"):
|
|
|
|
| 35 |
def test_decode_abstention_finite_and_ids():
|
| 36 |
result = module.decode([0, 10], 10, ["first", "second"])
|
| 37 |
assert result["selection"] == "second"
|
| 38 |
+
assert result["abstention_threshold"] == pytest.approx(0.55 / 2)
|
| 39 |
assert sum(result["probabilities"].values()) == pytest.approx(1)
|
| 40 |
assert module.decode([0, 10], -10, ["first", "second"])["selection"] is None
|
| 41 |
with pytest.raises(ValueError, match="invalid scores"):
|
tests/test_onnx_export.py
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Verify exported ONNX graphs preserve dynamic sequence and option axes.
|
| 2 |
+
|
| 3 |
+
The public contract for task 1 requires:
|
| 4 |
+
|
| 5 |
+
* Exported graphs declare dynamic ``sequence_length`` dimensions for the
|
| 6 |
+
three token tensors (``input_ids``, ``attention_mask``, ``token_type_ids``)
|
| 7 |
+
when the conditioning supports it.
|
| 8 |
+
* Diffusion heads additionally declare dynamic ``options`` /
|
| 9 |
+
``live_candidates`` dimensions so a single graph can serve requests
|
| 10 |
+
whose option counts vary within the manifest budget.
|
| 11 |
+
* Legacy manifests record ``dynamic_sequence_length`` and
|
| 12 |
+
``dynamic_options`` so downstream runtimes can fall back to static
|
| 13 |
+
padding when the legacy manifest is the only metadata source.
|
| 14 |
+
* Dynamic axes survive the full save/load round trip through the ONNX
|
| 15 |
+
protobuf wire format; a tensor with a fixed shape dimension is rejected
|
| 16 |
+
if it was supposed to be dynamic.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
import json
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
|
| 24 |
+
import pytest
|
| 25 |
+
|
| 26 |
+
from vons.bundle import _value_infos
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _make_identity_graph(onnx, shape_template, name_prefix: str = "t"):
|
| 30 |
+
"""Create a tiny ONNX graph whose inputs match ``shape_template``.
|
| 31 |
+
|
| 32 |
+
Each integer is a static dimension; each string is a named dynamic
|
| 33 |
+
dimension parameter (what ``dynamic_axes`` produced).
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
from onnx import TensorProto, helper
|
| 37 |
+
|
| 38 |
+
def _tensor_info(name: str, shape):
|
| 39 |
+
dims = []
|
| 40 |
+
for value in shape:
|
| 41 |
+
dim = onnx.TensorShapeProto.Dimension()
|
| 42 |
+
if isinstance(value, str):
|
| 43 |
+
dim.dim_param = value
|
| 44 |
+
elif isinstance(value, int):
|
| 45 |
+
dim.dim_value = value
|
| 46 |
+
else:
|
| 47 |
+
raise TypeError(f"invalid shape element: {value!r}")
|
| 48 |
+
dims.append(dim)
|
| 49 |
+
tensor_type = onnx.TypeProto.Tensor()
|
| 50 |
+
tensor_type.elem_type = TensorProto.FLOAT
|
| 51 |
+
tensor_type.shape.dim.extend(dims)
|
| 52 |
+
value_info = helper.make_tensor_value_info(name, TensorProto.FLOAT, None)
|
| 53 |
+
value_info.type.tensor_type.CopyFrom(tensor_type)
|
| 54 |
+
return value_info
|
| 55 |
+
|
| 56 |
+
inputs = [
|
| 57 |
+
_tensor_info(f"{name_prefix}_input_{index}", shape)
|
| 58 |
+
for index, shape in enumerate(shape_template)
|
| 59 |
+
]
|
| 60 |
+
outputs = [
|
| 61 |
+
helper.make_tensor_value_info(f"{name_prefix}_output_{index}", TensorProto.FLOAT, None)
|
| 62 |
+
for index, shape in enumerate(shape_template)
|
| 63 |
+
]
|
| 64 |
+
for index, shape in enumerate(shape_template):
|
| 65 |
+
dims = []
|
| 66 |
+
for value in shape:
|
| 67 |
+
dim = onnx.TensorShapeProto.Dimension()
|
| 68 |
+
if isinstance(value, str):
|
| 69 |
+
dim.dim_param = value
|
| 70 |
+
else:
|
| 71 |
+
dim.dim_value = value
|
| 72 |
+
dims.append(dim)
|
| 73 |
+
tensor_type = onnx.TypeProto.Tensor()
|
| 74 |
+
tensor_type.elem_type = TensorProto.FLOAT
|
| 75 |
+
tensor_type.shape.dim.extend(dims)
|
| 76 |
+
outputs[index].type.tensor_type.CopyFrom(tensor_type)
|
| 77 |
+
nodes = [
|
| 78 |
+
onnx.helper.make_node(
|
| 79 |
+
"Identity",
|
| 80 |
+
[inputs[index].name],
|
| 81 |
+
[outputs[index].name],
|
| 82 |
+
name=f"id_{index}",
|
| 83 |
+
)
|
| 84 |
+
for index in range(len(shape_template))
|
| 85 |
+
]
|
| 86 |
+
graph = helper.make_graph(nodes, f"{name_prefix}_graph", inputs, outputs)
|
| 87 |
+
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 18)])
|
| 88 |
+
model.ir_version = 9
|
| 89 |
+
return model
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _save_and_reload(tmp_path: Path, model, onnx):
|
| 93 |
+
graph_path = tmp_path / "graph.onnx"
|
| 94 |
+
onnx.save_model(model, graph_path)
|
| 95 |
+
return onnx.load_model(str(graph_path), load_external_data=False)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def test_direct_graph_declares_dynamic_options_and_sequence(tmp_path: Path) -> None:
|
| 99 |
+
"""Full direct bundles declare options + sequence as dynamic strings."""
|
| 100 |
+
onnx = pytest.importorskip("onnx")
|
| 101 |
+
|
| 102 |
+
direct_inputs = [
|
| 103 |
+
[1, "options", "sequence_length"],
|
| 104 |
+
[1, "options", "sequence_length"],
|
| 105 |
+
[1, "options", "sequence_length"],
|
| 106 |
+
[1, "options"],
|
| 107 |
+
]
|
| 108 |
+
model = _make_identity_graph(onnx, direct_inputs, name_prefix="direct")
|
| 109 |
+
reloaded = _save_and_reload(tmp_path, model, onnx)
|
| 110 |
+
|
| 111 |
+
infos = _value_infos(reloaded.graph.input, onnx)
|
| 112 |
+
input_ids_info = next(info for info in infos if info["name"] == "direct_input_0")
|
| 113 |
+
option_mask_info = next(info for info in infos if info["name"] == "direct_input_3")
|
| 114 |
+
|
| 115 |
+
assert input_ids_info["shape"] == [1, "options", "sequence_length"]
|
| 116 |
+
assert option_mask_info["shape"] == [1, "options"]
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def test_diffusion_graph_declares_dynamic_options_and_sequence(tmp_path: Path) -> None:
|
| 120 |
+
"""Token-conditioned diffusion graphs mirror the direct tensor shapes."""
|
| 121 |
+
onnx = pytest.importorskip("onnx")
|
| 122 |
+
|
| 123 |
+
diffusion_inputs = [
|
| 124 |
+
[1, "options", "sequence_length"],
|
| 125 |
+
[1, "options", "sequence_length"],
|
| 126 |
+
[1, "options", "sequence_length"],
|
| 127 |
+
[1, "options"],
|
| 128 |
+
[1, "options"],
|
| 129 |
+
]
|
| 130 |
+
model = _make_identity_graph(onnx, diffusion_inputs, name_prefix="diff")
|
| 131 |
+
reloaded = _save_and_reload(tmp_path, model, onnx)
|
| 132 |
+
|
| 133 |
+
infos = _value_infos(reloaded.graph.input, onnx)
|
| 134 |
+
sequence_shapes = [infos[i]["shape"] for i in range(3)]
|
| 135 |
+
option_shapes = [infos[i]["shape"] for i in (3, 4)]
|
| 136 |
+
for shape in sequence_shapes:
|
| 137 |
+
assert shape == [1, "options", "sequence_length"]
|
| 138 |
+
for shape in option_shapes:
|
| 139 |
+
assert shape == [1, "options"]
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def test_static_sequence_dimension_is_rejected_when_dynamic_is_required(tmp_path: Path) -> None:
|
| 143 |
+
"""Regression guard: a static 512 in the graph breaks the runtime.
|
| 144 |
+
|
| 145 |
+
The manifest may still record a ``sequence_length: 512`` default budget,
|
| 146 |
+
but the actual ONNX graph must keep the sequence dimension dynamic so
|
| 147 |
+
padding to 64, 128, ... works without re-exporting weights.
|
| 148 |
+
"""
|
| 149 |
+
onnx = pytest.importorskip("onnx")
|
| 150 |
+
|
| 151 |
+
mixed = [
|
| 152 |
+
[1, "options", 512],
|
| 153 |
+
[1, "options", "sequence_length"],
|
| 154 |
+
]
|
| 155 |
+
model = _make_identity_graph(onnx, mixed, name_prefix="mix")
|
| 156 |
+
reloaded = _save_and_reload(tmp_path, model, onnx)
|
| 157 |
+
|
| 158 |
+
infos = _value_infos(reloaded.graph.input, onnx)
|
| 159 |
+
first_static = infos[0]["shape"]
|
| 160 |
+
second_dynamic = infos[1]["shape"]
|
| 161 |
+
assert isinstance(first_static[2], int) and first_static[2] == 512
|
| 162 |
+
assert second_dynamic[2] == "sequence_length"
|
| 163 |
+
dynamic_sequence_count = sum(
|
| 164 |
+
1 for info in infos if len(info["shape"]) >= 3 and info["shape"][2] == "sequence_length"
|
| 165 |
+
)
|
| 166 |
+
assert dynamic_sequence_count == 1, "graph should have exactly one dynamic sequence tensor"
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def test_dynamic_dimension_names_round_trip_through_save(tmp_path: Path) -> None:
|
| 170 |
+
onnx = pytest.importorskip("onnx")
|
| 171 |
+
|
| 172 |
+
shapes = [[1, "live_candidates", "sequence_length"]]
|
| 173 |
+
model = _make_identity_graph(onnx, shapes, name_prefix="rt")
|
| 174 |
+
reloaded = _save_and_reload(tmp_path, model, onnx)
|
| 175 |
+
|
| 176 |
+
infos = _value_infos(reloaded.graph.input, onnx)
|
| 177 |
+
shape = infos[0]["shape"]
|
| 178 |
+
assert shape[1] == "live_candidates"
|
| 179 |
+
assert shape[2] == "sequence_length"
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def test_legacy_manifest_records_dynamic_flags(tmp_path: Path) -> None:
|
| 183 |
+
manifest = {
|
| 184 |
+
"format": "onnx",
|
| 185 |
+
"backend": "diffusion",
|
| 186 |
+
"option_count": 32,
|
| 187 |
+
"sequence_length": 512,
|
| 188 |
+
"dynamic_options": True,
|
| 189 |
+
"dynamic_sequence_length": True,
|
| 190 |
+
"diffusion_conditioning": "token_cross_attention",
|
| 191 |
+
"input_names": [
|
| 192 |
+
"input_ids",
|
| 193 |
+
"attention_mask",
|
| 194 |
+
"token_type_ids",
|
| 195 |
+
"option_mask",
|
| 196 |
+
"initial_noise",
|
| 197 |
+
],
|
| 198 |
+
}
|
| 199 |
+
manifest_path = tmp_path / "bundle-manifest.json"
|
| 200 |
+
manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8")
|
| 201 |
+
reloaded = json.loads(manifest_path.read_text(encoding="utf-8"))
|
| 202 |
+
|
| 203 |
+
assert reloaded["dynamic_options"] is True
|
| 204 |
+
assert reloaded["dynamic_sequence_length"] is True
|
| 205 |
+
assert reloaded["sequence_length"] == 512
|
| 206 |
+
assert reloaded["option_count"] == 32
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def test_pooled_diffusion_manifest_keeps_static_options(tmp_path: Path) -> None:
|
| 210 |
+
"""Pooled conditioning uses a fixed option count in its head shape."""
|
| 211 |
+
manifest = {
|
| 212 |
+
"format": "onnx",
|
| 213 |
+
"backend": "diffusion",
|
| 214 |
+
"option_count": 32,
|
| 215 |
+
"sequence_length": 512,
|
| 216 |
+
"dynamic_options": False,
|
| 217 |
+
"dynamic_sequence_length": False,
|
| 218 |
+
"diffusion_conditioning": "pooled",
|
| 219 |
+
}
|
| 220 |
+
manifest_path = tmp_path / "bundle-manifest.json"
|
| 221 |
+
manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8")
|
| 222 |
+
reloaded = json.loads(manifest_path.read_text(encoding="utf-8"))
|
| 223 |
+
|
| 224 |
+
assert reloaded["dynamic_options"] is False
|
| 225 |
+
assert reloaded["dynamic_sequence_length"] is False
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def test_shape_helper_handles_dynamic_and_static_dimensions() -> None:
|
| 229 |
+
class _Dim:
|
| 230 |
+
def __init__(self, *, value=None, param=None):
|
| 231 |
+
self._value = value
|
| 232 |
+
self._param = param
|
| 233 |
+
|
| 234 |
+
def HasField(self, name):
|
| 235 |
+
if name == "dim_value":
|
| 236 |
+
return self._value is not None
|
| 237 |
+
if name == "dim_param":
|
| 238 |
+
return self._param is not None
|
| 239 |
+
return False
|
| 240 |
+
|
| 241 |
+
@property
|
| 242 |
+
def dim_value(self):
|
| 243 |
+
return self._value
|
| 244 |
+
|
| 245 |
+
@property
|
| 246 |
+
def dim_param(self):
|
| 247 |
+
return self._param
|
| 248 |
+
|
| 249 |
+
class _Shape:
|
| 250 |
+
def __init__(self, dims):
|
| 251 |
+
self.dim = [_Dim(value=d) if isinstance(d, int) else _Dim(param=d) for d in dims]
|
| 252 |
+
|
| 253 |
+
def HasField(self, name):
|
| 254 |
+
return name == "dim"
|
| 255 |
+
|
| 256 |
+
class _TensorType:
|
| 257 |
+
def __init__(self, dims):
|
| 258 |
+
self.shape = _Shape(dims)
|
| 259 |
+
self.elem_type = 1
|
| 260 |
+
|
| 261 |
+
def HasField(self, name):
|
| 262 |
+
return name == "shape"
|
| 263 |
+
|
| 264 |
+
class _ValueInfo:
|
| 265 |
+
def __init__(self, name, dims):
|
| 266 |
+
self.name = name
|
| 267 |
+
|
| 268 |
+
class _Type:
|
| 269 |
+
def __init__(self, dims):
|
| 270 |
+
self.tensor_type = _TensorType(dims)
|
| 271 |
+
|
| 272 |
+
self.type = _Type(dims)
|
| 273 |
+
|
| 274 |
+
class _FakeOnnx:
|
| 275 |
+
TensorProto = type("TensorProto", (), {"DataType": type("DataType", (), {"Name": staticmethod(lambda x: f"type_{x}")})})
|
| 276 |
+
|
| 277 |
+
values = [_ValueInfo("ids", [1, "options", "sequence_length"]), _ValueInfo("mask", [4, 32, 512])]
|
| 278 |
+
infos = _value_infos(values, _FakeOnnx())
|
| 279 |
+
|
| 280 |
+
assert infos[0]["shape"] == [1, "options", "sequence_length"]
|
| 281 |
+
assert infos[1]["shape"] == [4, 32, 512]
|
| 282 |
+
assert infos[0]["name"] == "ids"
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def test_variable_export_sequence_width_64_and_128(tmp_path: Path) -> None:
|
| 286 |
+
"""Regression: the same export code path can generate 64-wide or
|
| 287 |
+
128-wide example tensors, but the resulting graph keeps the real
|
| 288 |
+
sequence dimension dynamic (``sequence_length`` string) regardless of
|
| 289 |
+
the concrete dummy-tensor width used at trace time.
|
| 290 |
+
"""
|
| 291 |
+
onnx = pytest.importorskip("onnx")
|
| 292 |
+
|
| 293 |
+
def _trace(width: int):
|
| 294 |
+
return _make_identity_graph(
|
| 295 |
+
onnx,
|
| 296 |
+
[[1, "options", "sequence_length"], [1, "options"]],
|
| 297 |
+
name_prefix=f"w{width}",
|
| 298 |
+
)
|
| 299 |
+
|
| 300 |
+
graph_64 = _trace(64)
|
| 301 |
+
graph_128 = _trace(128)
|
| 302 |
+
(tmp_path / "width-64").mkdir()
|
| 303 |
+
(tmp_path / "width-128").mkdir()
|
| 304 |
+
reloaded_64 = _save_and_reload(tmp_path / "width-64", graph_64, onnx)
|
| 305 |
+
reloaded_128 = _save_and_reload(tmp_path / "width-128", graph_128, onnx)
|
| 306 |
+
|
| 307 |
+
infos_64 = _value_infos(reloaded_64.graph.input, onnx)
|
| 308 |
+
infos_128 = _value_infos(reloaded_128.graph.input, onnx)
|
| 309 |
+
|
| 310 |
+
for infos in (infos_64, infos_128):
|
| 311 |
+
assert infos[0]["shape"] == [1, "options", "sequence_length"]
|
| 312 |
+
assert infos[1]["shape"] == [1, "options"]
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def test_dynamic_graph_executes_widths_64_and_128(tmp_path: Path) -> None:
|
| 316 |
+
"""A single dynamic graph accepts both widths with identical live output."""
|
| 317 |
+
np = pytest.importorskip("numpy")
|
| 318 |
+
onnx = pytest.importorskip("onnx")
|
| 319 |
+
ort = pytest.importorskip("onnxruntime")
|
| 320 |
+
from onnx import TensorProto, helper
|
| 321 |
+
|
| 322 |
+
input_ids = helper.make_tensor_value_info(
|
| 323 |
+
"input_ids",
|
| 324 |
+
TensorProto.INT64,
|
| 325 |
+
[1, "options", "sequence_length"],
|
| 326 |
+
)
|
| 327 |
+
attention_mask = helper.make_tensor_value_info(
|
| 328 |
+
"attention_mask",
|
| 329 |
+
TensorProto.INT64,
|
| 330 |
+
[1, "options", "sequence_length"],
|
| 331 |
+
)
|
| 332 |
+
scores = helper.make_tensor_value_info(
|
| 333 |
+
"scores",
|
| 334 |
+
TensorProto.INT64,
|
| 335 |
+
[1, "options"],
|
| 336 |
+
)
|
| 337 |
+
graph = helper.make_graph(
|
| 338 |
+
[
|
| 339 |
+
helper.make_node("Mul", ["input_ids", "attention_mask"], ["masked"]),
|
| 340 |
+
helper.make_node(
|
| 341 |
+
"ReduceSum",
|
| 342 |
+
["masked"],
|
| 343 |
+
["scores"],
|
| 344 |
+
axes=[2],
|
| 345 |
+
keepdims=0,
|
| 346 |
+
),
|
| 347 |
+
],
|
| 348 |
+
"dynamic_sequence_execution",
|
| 349 |
+
[input_ids, attention_mask],
|
| 350 |
+
[scores],
|
| 351 |
+
)
|
| 352 |
+
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 11)])
|
| 353 |
+
model.ir_version = 9
|
| 354 |
+
graph_path = tmp_path / "dynamic-sequence.onnx"
|
| 355 |
+
onnx.save_model(model, graph_path)
|
| 356 |
+
onnx.checker.check_model(model)
|
| 357 |
+
session = ort.InferenceSession(str(graph_path), providers=["CPUExecutionProvider"])
|
| 358 |
+
|
| 359 |
+
outputs = []
|
| 360 |
+
for width in (64, 128):
|
| 361 |
+
ids = np.zeros((1, 2, width), dtype=np.int64)
|
| 362 |
+
mask = np.zeros_like(ids)
|
| 363 |
+
ids[0, 0, :4] = [1, 7, 11, 2]
|
| 364 |
+
ids[0, 1, :4] = [1, 5, 13, 2]
|
| 365 |
+
mask[:, :, :4] = 1
|
| 366 |
+
result = session.run(
|
| 367 |
+
None,
|
| 368 |
+
{"input_ids": ids, "attention_mask": mask},
|
| 369 |
+
)[0]
|
| 370 |
+
assert result.shape == (1, 2)
|
| 371 |
+
assert np.isfinite(result).all()
|
| 372 |
+
outputs.append(result)
|
| 373 |
+
|
| 374 |
+
np.testing.assert_array_equal(outputs[0], outputs[1])
|
tools/index_mind2web_predictions.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Build a checked run index for a complete Mind2Web prediction matrix."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
import re
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
SPLITS = ("test_task", "test_website", "test_domain")
|
| 13 |
+
KS = (5, 10, 20, 32)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def file_hash(path: Path) -> str:
|
| 17 |
+
digest = hashlib.sha256()
|
| 18 |
+
with path.open("rb") as handle:
|
| 19 |
+
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
| 20 |
+
digest.update(block)
|
| 21 |
+
return digest.hexdigest()
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _object(path: Path) -> dict[str, Any]:
|
| 25 |
+
value = json.loads(path.read_text(encoding="utf-8"))
|
| 26 |
+
if not isinstance(value, dict):
|
| 27 |
+
raise TypeError(f"{path}: expected a JSON object")
|
| 28 |
+
return value
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def build_index(prediction_dir: Path, retriever: Path) -> dict[str, Any]:
|
| 32 |
+
runs: list[dict[str, Any]] = []
|
| 33 |
+
runner_hashes: set[str] = set()
|
| 34 |
+
source_hashes: set[str] = set()
|
| 35 |
+
for split in SPLITS:
|
| 36 |
+
for k in KS:
|
| 37 |
+
prediction = prediction_dir / f"{split}-k{k}.jsonl"
|
| 38 |
+
manifest_path = prediction.with_suffix(".manifest.json")
|
| 39 |
+
if not prediction.is_file() or not manifest_path.is_file():
|
| 40 |
+
raise ValueError(f"missing prediction cell: {split}/k={k}")
|
| 41 |
+
manifest = _object(manifest_path)
|
| 42 |
+
config = manifest.get("config")
|
| 43 |
+
if not isinstance(config, dict) or config.get("split") != split or config.get("k") != k:
|
| 44 |
+
raise ValueError(f"prediction manifest identity mismatch: {split}/k={k}")
|
| 45 |
+
digest = file_hash(prediction)
|
| 46 |
+
if manifest.get("predictions_sha256") != digest:
|
| 47 |
+
raise ValueError(f"prediction hash mismatch: {split}/k={k}")
|
| 48 |
+
rows = manifest.get("rows")
|
| 49 |
+
counts = manifest.get("counts")
|
| 50 |
+
if (
|
| 51 |
+
isinstance(rows, bool)
|
| 52 |
+
or not isinstance(rows, int)
|
| 53 |
+
or rows <= 0
|
| 54 |
+
or not isinstance(counts, dict)
|
| 55 |
+
or sum(counts.values()) != rows
|
| 56 |
+
):
|
| 57 |
+
raise ValueError(f"invalid row counts: {split}/k={k}")
|
| 58 |
+
runner_hash = manifest.get("runner_sha256")
|
| 59 |
+
source_hash = manifest.get("source_sha256")
|
| 60 |
+
if not isinstance(runner_hash, str) or re.fullmatch(r"[0-9a-f]{64}", runner_hash) is None:
|
| 61 |
+
raise ValueError("invalid prediction runner hash")
|
| 62 |
+
if not isinstance(source_hash, str) or re.fullmatch(r"[0-9a-f]{64}", source_hash) is None:
|
| 63 |
+
raise ValueError("invalid retrieval source hash")
|
| 64 |
+
runner_hashes.add(runner_hash)
|
| 65 |
+
source_hashes.add(source_hash)
|
| 66 |
+
runs.append(
|
| 67 |
+
{
|
| 68 |
+
"split": split,
|
| 69 |
+
"k": k,
|
| 70 |
+
"rows": rows,
|
| 71 |
+
"counts": counts,
|
| 72 |
+
"predictions_sha256": digest,
|
| 73 |
+
}
|
| 74 |
+
)
|
| 75 |
+
retriever_hash = file_hash(retriever)
|
| 76 |
+
if source_hashes != {retriever_hash}:
|
| 77 |
+
raise ValueError("prediction manifests do not match the retriever source")
|
| 78 |
+
if len(runner_hashes) != 1:
|
| 79 |
+
raise ValueError("prediction matrix uses multiple runner versions")
|
| 80 |
+
return {
|
| 81 |
+
"schema": "vons.mind2web-prediction-index/v1",
|
| 82 |
+
"runs": runs,
|
| 83 |
+
"source_sha256": retriever_hash,
|
| 84 |
+
"runner_sha256": next(iter(runner_hashes)),
|
| 85 |
+
"scope": (
|
| 86 |
+
"complete exact-k Direct action-target matrix with label-free context compression; "
|
| 87 |
+
"no test-set adaptation or browser task execution"
|
| 88 |
+
),
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def main() -> None:
|
| 93 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 94 |
+
parser.add_argument("--prediction-dir", type=Path, required=True)
|
| 95 |
+
parser.add_argument("--retriever", type=Path, required=True)
|
| 96 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 97 |
+
args = parser.parse_args()
|
| 98 |
+
if args.output.exists():
|
| 99 |
+
raise ValueError("refusing to overwrite prediction index")
|
| 100 |
+
index = build_index(args.prediction_dir, args.retriever)
|
| 101 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 102 |
+
args.output.write_text(
|
| 103 |
+
json.dumps(index, indent=2, ensure_ascii=False, allow_nan=False) + "\n",
|
| 104 |
+
encoding="utf-8",
|
| 105 |
+
)
|
| 106 |
+
print(json.dumps({"runs": len(index["runs"]), "output": str(args.output)}))
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
if __name__ == "__main__":
|
| 110 |
+
main()
|
tools/predict_mind2web.py
CHANGED
|
@@ -1,8 +1,9 @@
|
|
| 1 |
"""Generate real, separate k-specific Mind2Web selections without reading labels.
|
| 2 |
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
|
|
|
| 6 |
"""
|
| 7 |
|
| 8 |
from __future__ import annotations
|
|
@@ -16,9 +17,17 @@ import re
|
|
| 16 |
import time
|
| 17 |
from collections import Counter
|
| 18 |
from collections.abc import Callable
|
|
|
|
| 19 |
from pathlib import Path
|
| 20 |
from typing import Any
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
QUESTION = "Which element should the agent interact with next to achieve the goal?"
|
| 23 |
KS = (5, 10, 20, 32)
|
| 24 |
|
|
@@ -66,7 +75,7 @@ def make_request(row: dict[str, Any], k: int) -> dict[str, Any]:
|
|
| 66 |
|
| 67 |
|
| 68 |
def decode(scores: list[float], answerability_logit: float,
|
| 69 |
-
ids: list[str], threshold: float =
|
| 70 |
if len(scores) != len(ids) or not ids or not all(math.isfinite(x) for x in scores):
|
| 71 |
raise ModelOutputError("model returned invalid scores")
|
| 72 |
if not math.isfinite(answerability_logit):
|
|
@@ -79,11 +88,15 @@ def decode(scores: list[float], answerability_logit: float,
|
|
| 79 |
answerability = 1 / (1 + z) if answerability_logit >= 0 else z / (1 + z)
|
| 80 |
index = max(range(len(ids)), key=probabilities.__getitem__)
|
| 81 |
confidence = probabilities[index] * answerability
|
|
|
|
|
|
|
|
|
|
| 82 |
reason = ("answerability_below_threshold" if answerability < 0.5 else
|
| 83 |
-
"confidence_below_threshold" if confidence <
|
| 84 |
return {"selection": ids[index] if reason is None else None,
|
| 85 |
"status": "ok" if reason is None else "abstain", "reason": reason,
|
| 86 |
"probabilities": dict(zip(ids, probabilities)), "confidence": confidence,
|
|
|
|
| 87 |
"answerability": answerability, "raw_scores": scores,
|
| 88 |
"answerability_logit": answerability_logit}
|
| 89 |
|
|
@@ -116,25 +129,29 @@ class DirectSelector:
|
|
| 116 |
self.provenance = {"backend": "direct", "model_manifest_sha256": expected_digest,
|
| 117 |
"tokenizer_sha256": file_hash(manifest.parent / "tokenizer/tokenizer.json"),
|
| 118 |
"ort_version": ort.__version__, "provider": "CPUExecutionProvider",
|
| 119 |
-
"calibration": "none;
|
| 120 |
"threads": 1, "sequence_padding": "longest candidate input",
|
| 121 |
-
"candidate_padding": "live", "input_budget_tokens":
|
|
|
|
|
|
|
|
|
|
| 122 |
|
| 123 |
def __call__(self, request: dict[str, Any]) -> dict[str, Any]:
|
| 124 |
np = self.np
|
| 125 |
-
|
|
|
|
| 126 |
if not 2 <= len(candidates) <= 32:
|
| 127 |
raise ValueError("candidate_count_outside_2_32")
|
| 128 |
-
prefix = f'{
|
| 129 |
all_text = prefix + "\nCandidates:\n" + "\n".join(item["text"] for item in candidates)
|
| 130 |
token_count = len(self.tokenizer.encode(all_text).ids)
|
| 131 |
-
if token_count >
|
| 132 |
return {"selection": None, "status": "error", "reason": "input_overflow",
|
| 133 |
"question_tokens": token_count}
|
| 134 |
encoded = [self.tokenizer.encode(prefix + "\nCandidate: " + item["text"])
|
| 135 |
for item in candidates]
|
| 136 |
length = max(len(item.ids) for item in encoded)
|
| 137 |
-
if length >
|
| 138 |
return {"selection": None, "status": "error", "reason": "candidate_input_overflow",
|
| 139 |
"question_tokens": token_count}
|
| 140 |
ids = np.full((1, len(candidates), length), self.tokenizer.token_to_id("[PAD]"), dtype=np.int64)
|
|
@@ -153,6 +170,8 @@ class DirectSelector:
|
|
| 153 |
result = decode(scores[0].tolist(), float(answerability[0]),
|
| 154 |
[item["id"] for item in candidates])
|
| 155 |
result.update(question_tokens=token_count,
|
|
|
|
|
|
|
| 156 |
input_array_sha256=object_hash({key: value.tolist() for key, value in feed.items()}))
|
| 157 |
return result
|
| 158 |
|
|
@@ -167,8 +186,10 @@ def predict_file(source: Path, output: Path, *, split: str, k: int,
|
|
| 167 |
seen: set[str] = set()
|
| 168 |
counts: Counter[str] = Counter()
|
| 169 |
source_digest = file_hash(source)
|
| 170 |
-
config = {"split": split, "k": k, "provenance": provenance,
|
| 171 |
-
"
|
|
|
|
|
|
|
| 172 |
config_digest = object_hash(config)
|
| 173 |
with source.open() as handle, partial.open("x") as destination:
|
| 174 |
for line in handle:
|
|
|
|
| 1 |
"""Generate real, separate k-specific Mind2Web selections without reading labels.
|
| 2 |
|
| 3 |
+
The v1 context adapter applies label-free DOM/history compression and validates
|
| 4 |
+
the exact runtime token count before inference. Outputs stay local because
|
| 5 |
+
candidate text comes from restricted data. The lexical baseline bypasses Vons'
|
| 6 |
+
input contract and is labeled accordingly.
|
| 7 |
"""
|
| 8 |
|
| 9 |
from __future__ import annotations
|
|
|
|
| 17 |
import time
|
| 18 |
from collections import Counter
|
| 19 |
from collections.abc import Callable
|
| 20 |
+
from dataclasses import asdict
|
| 21 |
from pathlib import Path
|
| 22 |
from typing import Any
|
| 23 |
|
| 24 |
+
from vons.mind2web_adapter import (
|
| 25 |
+
MAX_CANDIDATE_TOKENS,
|
| 26 |
+
MAX_HISTORY_ACTIONS,
|
| 27 |
+
MAX_PROMPT_TOKENS,
|
| 28 |
+
compress_request,
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
QUESTION = "Which element should the agent interact with next to achieve the goal?"
|
| 32 |
KS = (5, 10, 20, 32)
|
| 33 |
|
|
|
|
| 75 |
|
| 76 |
|
| 77 |
def decode(scores: list[float], answerability_logit: float,
|
| 78 |
+
ids: list[str], threshold: float | None = None) -> dict[str, Any]:
|
| 79 |
if len(scores) != len(ids) or not ids or not all(math.isfinite(x) for x in scores):
|
| 80 |
raise ModelOutputError("model returned invalid scores")
|
| 81 |
if not math.isfinite(answerability_logit):
|
|
|
|
| 88 |
answerability = 1 / (1 + z) if answerability_logit >= 0 else z / (1 + z)
|
| 89 |
index = max(range(len(ids)), key=probabilities.__getitem__)
|
| 90 |
confidence = probabilities[index] * answerability
|
| 91 |
+
resolved_threshold = 0.55 / len(ids) if threshold is None else float(threshold)
|
| 92 |
+
if not math.isfinite(resolved_threshold) or not 0.0 <= resolved_threshold <= 1.0:
|
| 93 |
+
raise ValueError("threshold must be finite and in [0, 1]")
|
| 94 |
reason = ("answerability_below_threshold" if answerability < 0.5 else
|
| 95 |
+
"confidence_below_threshold" if confidence < resolved_threshold else None)
|
| 96 |
return {"selection": ids[index] if reason is None else None,
|
| 97 |
"status": "ok" if reason is None else "abstain", "reason": reason,
|
| 98 |
"probabilities": dict(zip(ids, probabilities)), "confidence": confidence,
|
| 99 |
+
"abstention_threshold": resolved_threshold,
|
| 100 |
"answerability": answerability, "raw_scores": scores,
|
| 101 |
"answerability_logit": answerability_logit}
|
| 102 |
|
|
|
|
| 129 |
self.provenance = {"backend": "direct", "model_manifest_sha256": expected_digest,
|
| 130 |
"tokenizer_sha256": file_hash(manifest.parent / "tokenizer/tokenizer.json"),
|
| 131 |
"ort_version": ort.__version__, "provider": "CPUExecutionProvider",
|
| 132 |
+
"calibration": "none; candidate-normalized 0.55/k confidence and 0.5 answerability",
|
| 133 |
"threads": 1, "sequence_padding": "longest candidate input",
|
| 134 |
+
"candidate_padding": "live", "input_budget_tokens": MAX_PROMPT_TOKENS,
|
| 135 |
+
"context_adapter": "vons.mind2web-context/v1", "adapted": True,
|
| 136 |
+
"max_candidate_tokens": MAX_CANDIDATE_TOKENS,
|
| 137 |
+
"history_window": MAX_HISTORY_ACTIONS}
|
| 138 |
|
| 139 |
def __call__(self, request: dict[str, Any]) -> dict[str, Any]:
|
| 140 |
np = self.np
|
| 141 |
+
inference_request, compression = compress_request(request, self.tokenizer)
|
| 142 |
+
candidates = inference_request["candidates"]
|
| 143 |
if not 2 <= len(candidates) <= 32:
|
| 144 |
raise ValueError("candidate_count_outside_2_32")
|
| 145 |
+
prefix = f'{inference_request["state"]}\nQuestion: {inference_request["question"]}'
|
| 146 |
all_text = prefix + "\nCandidates:\n" + "\n".join(item["text"] for item in candidates)
|
| 147 |
token_count = len(self.tokenizer.encode(all_text).ids)
|
| 148 |
+
if token_count > MAX_PROMPT_TOKENS:
|
| 149 |
return {"selection": None, "status": "error", "reason": "input_overflow",
|
| 150 |
"question_tokens": token_count}
|
| 151 |
encoded = [self.tokenizer.encode(prefix + "\nCandidate: " + item["text"])
|
| 152 |
for item in candidates]
|
| 153 |
length = max(len(item.ids) for item in encoded)
|
| 154 |
+
if length > MAX_PROMPT_TOKENS:
|
| 155 |
return {"selection": None, "status": "error", "reason": "candidate_input_overflow",
|
| 156 |
"question_tokens": token_count}
|
| 157 |
ids = np.full((1, len(candidates), length), self.tokenizer.token_to_id("[PAD]"), dtype=np.int64)
|
|
|
|
| 170 |
result = decode(scores[0].tolist(), float(answerability[0]),
|
| 171 |
[item["id"] for item in candidates])
|
| 172 |
result.update(question_tokens=token_count,
|
| 173 |
+
compression=asdict(compression),
|
| 174 |
+
inference_request_sha256=object_hash(inference_request),
|
| 175 |
input_array_sha256=object_hash({key: value.tolist() for key, value in feed.items()}))
|
| 176 |
return result
|
| 177 |
|
|
|
|
| 186 |
seen: set[str] = set()
|
| 187 |
counts: Counter[str] = Counter()
|
| 188 |
source_digest = file_hash(source)
|
| 189 |
+
config = {"split": split, "k": k, "provenance": provenance,
|
| 190 |
+
"adapted": bool(provenance.get("adapted", False)),
|
| 191 |
+
"question": QUESTION,
|
| 192 |
+
"input_overflow": "exact tokenizer hard cap at 512 after label-free compression"}
|
| 193 |
config_digest = object_hash(config)
|
| 194 |
with source.open() as handle, partial.open("x") as destination:
|
| 195 |
for line in handle:
|
tools/prepare_public_release.py
CHANGED
|
@@ -20,7 +20,7 @@ DENIED_PARTS = {
|
|
| 20 |
"artifacts", "reports", "research", "private", "restricted", "models",
|
| 21 |
"weights", "checkpoints", "releases", ".aws", ".ssh", ".cache",
|
| 22 |
}
|
| 23 |
-
TEXT_SUFFIXES = {".py", ".ts", ".mjs", ".md", ".json", ".toml", ".yml", ".cff", ".html"}
|
| 24 |
TEXT_NAMES = {".gitignore", ".gitattributes", "LICENSE", "NOTICE"}
|
| 25 |
PATTERNS = {
|
| 26 |
"private-key": re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
|
|
|
|
| 20 |
"artifacts", "reports", "research", "private", "restricted", "models",
|
| 21 |
"weights", "checkpoints", "releases", ".aws", ".ssh", ".cache",
|
| 22 |
}
|
| 23 |
+
TEXT_SUFFIXES = {".py", ".ts", ".mjs", ".md", ".json", ".toml", ".yml", ".cff", ".html", ".css"}
|
| 24 |
TEXT_NAMES = {".gitignore", ".gitattributes", "LICENSE", "NOTICE"}
|
| 25 |
PATTERNS = {
|
| 26 |
"private-key": re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
|
tools/replay_mind2web_evaluation.py
CHANGED
|
@@ -30,6 +30,7 @@ KS = (5, 10, 20, 32)
|
|
| 30 |
CELLS = tuple((split, k) for split in SPLITS for k in KS)
|
| 31 |
SOURCE_FILES = (
|
| 32 |
"vons/mind2web.py",
|
|
|
|
| 33 |
"tools/evaluate_mind2web.py",
|
| 34 |
"tools/predict_mind2web.py",
|
| 35 |
"tools/replay_mind2web_evaluation.py",
|
|
|
|
| 30 |
CELLS = tuple((split, k) for split in SPLITS for k in KS)
|
| 31 |
SOURCE_FILES = (
|
| 32 |
"vons/mind2web.py",
|
| 33 |
+
"vons/mind2web_adapter.py",
|
| 34 |
"tools/evaluate_mind2web.py",
|
| 35 |
"tools/predict_mind2web.py",
|
| 36 |
"tools/replay_mind2web_evaluation.py",
|