Publish Vons source preview and reviewed research PDFs
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +12 -35
- .github/ISSUE_TEMPLATE/bug.yml +31 -0
- .github/ISSUE_TEMPLATE/experience.yml +55 -0
- .github/ISSUE_TEMPLATE/reproduction.yml +50 -0
- .github/PULL_REQUEST_TEMPLATE.md +16 -0
- .github/workflows/checks.yml +61 -0
- .gitignore +148 -0
- CITATION.cff +11 -0
- CONTRIBUTING.md +81 -0
- LICENSE +104 -0
- README.md +91 -1
- configs/pilot-diffusion.json +16 -0
- configs/pilot.json +15 -0
- configs/public-release.json +111 -0
- docs/COMMUNITY.md +115 -0
- docs/HUB_PUBLISHING.md +118 -0
- docs/HUGGING_FACE.md +93 -0
- docs/KASI_INTEGRATION.md +44 -0
- docs/LICENSING.md +42 -0
- docs/MODEL_CARD.md +54 -0
- docs/PUBLIC_RELEASE.md +94 -0
- docs/assets/inlevel9-signature-dark.png +0 -0
- docs/assets/inlevel9-signature.png +0 -0
- docs/publications/ARXIV.md +67 -0
- docs/publications/LICENSE.md +40 -0
- docs/publications/README.md +40 -0
- docs/publications/vons-paper-v1.0.pdf +0 -0
- docs/publications/vons-tech-report-v1.0.1.pdf +0 -0
- pyproject.toml +35 -0
- sdk/typescript/LICENSE +104 -0
- sdk/typescript/README.md +49 -0
- sdk/typescript/demo/benchmark.html +17 -0
- sdk/typescript/demo/benchmark.ts +285 -0
- sdk/typescript/demo/index.html +20 -0
- sdk/typescript/demo/main.ts +78 -0
- sdk/typescript/demo/parity.html +20 -0
- sdk/typescript/demo/parity.ts +239 -0
- sdk/typescript/package-lock.json +652 -0
- sdk/typescript/package.json +33 -0
- sdk/typescript/src/index.ts +255 -0
- sdk/typescript/src/onnx.ts +606 -0
- sdk/typescript/test/index.test.mjs +65 -0
- sdk/typescript/test/onnx.test.mjs +101 -0
- sdk/typescript/test/parity.test.mjs +18 -0
- tests/test_browser_benchmark.py +161 -0
- tests/test_browser_benchmark_summary.py +1130 -0
- tests/test_browser_parity_reference.py +350 -0
- tests/test_bundle.py +136 -0
- tests/test_calibration.py +241 -0
- tests/test_contract.py +40 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,12 @@
|
|
| 1 |
-
*
|
| 2 |
-
*.
|
| 3 |
-
*.
|
| 4 |
-
*.
|
| 5 |
-
*.
|
| 6 |
-
*.
|
| 7 |
-
*.
|
| 8 |
-
*.
|
| 9 |
-
*.
|
| 10 |
-
*.
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
* text=auto
|
| 2 |
+
*.py text eol=lf
|
| 3 |
+
*.ts text eol=lf
|
| 4 |
+
*.mjs text eol=lf
|
| 5 |
+
*.md text eol=lf
|
| 6 |
+
*.json text eol=lf
|
| 7 |
+
*.yml text eol=lf
|
| 8 |
+
*.cff text eol=lf
|
| 9 |
+
*.pdf binary
|
| 10 |
+
*.png binary
|
| 11 |
+
docs/publications/vons-paper-v1.0.pdf filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
docs/publications/vons-tech-report-v1.0.1.pdf filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/ISSUE_TEMPLATE/bug.yml
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Bug report
|
| 2 |
+
description: Report unexpected behavior with a minimal safe reproduction.
|
| 3 |
+
title: "[Bug] "
|
| 4 |
+
body:
|
| 5 |
+
- type: markdown
|
| 6 |
+
attributes:
|
| 7 |
+
value: |
|
| 8 |
+
For security issues, contact oswarld@inlevel9.com privately with a brief,
|
| 9 |
+
nonsensitive description first. Do not post secrets or restricted inputs.
|
| 10 |
+
English and Korean reports are welcome.
|
| 11 |
+
- type: textarea
|
| 12 |
+
id: environment
|
| 13 |
+
attributes:
|
| 14 |
+
label: Version and environment
|
| 15 |
+
description: Source revision or release manifest, OS, runtime version and optional model/provider identity.
|
| 16 |
+
validations:
|
| 17 |
+
required: true
|
| 18 |
+
- type: textarea
|
| 19 |
+
id: reproduction
|
| 20 |
+
attributes:
|
| 21 |
+
label: Minimal reproduction
|
| 22 |
+
description: Exact command and preferably synthetic input; remove personal information and credentials.
|
| 23 |
+
validations:
|
| 24 |
+
required: true
|
| 25 |
+
- type: textarea
|
| 26 |
+
id: behavior
|
| 27 |
+
attributes:
|
| 28 |
+
label: Expected and actual behavior
|
| 29 |
+
description: Include safe error output, frequency and any workaround.
|
| 30 |
+
validations:
|
| 31 |
+
required: true
|
.github/ISSUE_TEMPLATE/experience.yml
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Experience report
|
| 2 |
+
description: Share what worked, failed or remained unclear when trying Vons.
|
| 3 |
+
title: "[Experience] "
|
| 4 |
+
body:
|
| 5 |
+
- type: markdown
|
| 6 |
+
attributes:
|
| 7 |
+
value: |
|
| 8 |
+
Positive and negative experiences are welcome, in English or Korean.
|
| 9 |
+
This is a public report once the repository is public. Use synthetic or
|
| 10 |
+
redacted examples. Do not upload secrets, private prompts, customer traces,
|
| 11 |
+
restricted data or model weights. A report is not benchmark validation.
|
| 12 |
+
- type: dropdown
|
| 13 |
+
id: path
|
| 14 |
+
attributes:
|
| 15 |
+
label: What did you try?
|
| 16 |
+
options:
|
| 17 |
+
- Documentation or installation
|
| 18 |
+
- Deterministic contract / KASI adapter
|
| 19 |
+
- Synthetic data walkthrough
|
| 20 |
+
- Model inference with separately supplied assets
|
| 21 |
+
validations:
|
| 22 |
+
required: true
|
| 23 |
+
- type: input
|
| 24 |
+
id: version
|
| 25 |
+
attributes:
|
| 26 |
+
label: Version or source identity
|
| 27 |
+
description: Commit, release manifest digest or explicitly unknown; include asset digest for inference.
|
| 28 |
+
validations:
|
| 29 |
+
required: true
|
| 30 |
+
- type: textarea
|
| 31 |
+
id: context
|
| 32 |
+
attributes:
|
| 33 |
+
label: Scenario and environment
|
| 34 |
+
description: Your goal, OS, Python or Node version, device, runtime and provider. Omit personal details.
|
| 35 |
+
validations:
|
| 36 |
+
required: true
|
| 37 |
+
- type: textarea
|
| 38 |
+
id: outcome
|
| 39 |
+
attributes:
|
| 40 |
+
label: Expected and observed behavior
|
| 41 |
+
description: Include the command or steps, what helped, what failed and limitations.
|
| 42 |
+
validations:
|
| 43 |
+
required: true
|
| 44 |
+
- type: textarea
|
| 45 |
+
id: counts
|
| 46 |
+
attributes:
|
| 47 |
+
label: Counts and optional safe evidence
|
| 48 |
+
description: Attempted / successful / abstained / failed with definitions; explain overlaps. Use unavailable where unmeasured. Link only reviewed evidence.
|
| 49 |
+
- type: checkboxes
|
| 50 |
+
id: privacy
|
| 51 |
+
attributes:
|
| 52 |
+
label: Public content check
|
| 53 |
+
options:
|
| 54 |
+
- label: I reviewed this report for secrets, personal information and restricted material, and may share its contents.
|
| 55 |
+
required: true
|
.github/ISSUE_TEMPLATE/reproduction.yml
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Reproduction report
|
| 2 |
+
description: Report a scoped attempt to reproduce a documented behavior or result.
|
| 3 |
+
title: "[Reproduction] "
|
| 4 |
+
body:
|
| 5 |
+
- type: markdown
|
| 6 |
+
attributes:
|
| 7 |
+
value: |
|
| 8 |
+
An unsuccessful reproduction is useful. Keep raw records locally and
|
| 9 |
+
publish only permitted, reviewed aggregates or synthetic examples.
|
| 10 |
+
English and Korean reports are welcome. Missing evidence is unavailable.
|
| 11 |
+
- type: textarea
|
| 12 |
+
id: claim
|
| 13 |
+
attributes:
|
| 14 |
+
label: Claim and source
|
| 15 |
+
description: Link the exact document/version and state the claim's scope and expected result.
|
| 16 |
+
validations:
|
| 17 |
+
required: true
|
| 18 |
+
- type: textarea
|
| 19 |
+
id: setup
|
| 20 |
+
attributes:
|
| 21 |
+
label: Versions, inputs and environment
|
| 22 |
+
description: Source revision or manifest, model/tokenizer hashes if used, permitted input provenance, split, seed, OS/device/runtime/provider. Do not attach restricted inputs.
|
| 23 |
+
validations:
|
| 24 |
+
required: true
|
| 25 |
+
- type: textarea
|
| 26 |
+
id: procedure
|
| 27 |
+
attributes:
|
| 28 |
+
label: Commands and measurement procedure
|
| 29 |
+
description: Include warm-up, repetitions, timing or memory scope and analysis choices where applicable.
|
| 30 |
+
validations:
|
| 31 |
+
required: true
|
| 32 |
+
- type: textarea
|
| 33 |
+
id: results
|
| 34 |
+
attributes:
|
| 35 |
+
label: Observed results and denominators
|
| 36 |
+
description: Attempts, successes, abstentions, failures and unavailable values; include definitions, mismatches and limitations. Separate targets from measurements.
|
| 37 |
+
validations:
|
| 38 |
+
required: true
|
| 39 |
+
- type: textarea
|
| 40 |
+
id: evidence
|
| 41 |
+
attributes:
|
| 42 |
+
label: Optional permitted evidence
|
| 43 |
+
description: Safe aggregate report or synthetic fixture with hashes; state what cannot be shared or independently checked.
|
| 44 |
+
- type: checkboxes
|
| 45 |
+
id: privacy
|
| 46 |
+
attributes:
|
| 47 |
+
label: Public content check
|
| 48 |
+
options:
|
| 49 |
+
- label: I reviewed the report and attachments for secrets, personal information and restricted material, and may share them.
|
| 50 |
+
required: true
|
.github/PULL_REQUEST_TEMPLATE.md
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## Problem and change
|
| 2 |
+
|
| 3 |
+
Describe the behavior before and after this change, and link the related issue.
|
| 4 |
+
|
| 5 |
+
## Verification
|
| 6 |
+
|
| 7 |
+
List commands run and their results. State skips, unavailable evidence and known
|
| 8 |
+
limitations. Separate measured values from proposed targets.
|
| 9 |
+
|
| 10 |
+
## Review notes
|
| 11 |
+
|
| 12 |
+
- Explain any public contract, dependency, data or license implications.
|
| 13 |
+
- Identify third-party material and its provenance when applicable.
|
| 14 |
+
- Keep raw restricted data, model weights, secrets and private prompts out.
|
| 15 |
+
- Update the release allowlist for reviewed new public files.
|
| 16 |
+
- Preserve the separation between model decisions and host execution/consent.
|
.github/workflows/checks.yml
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Source checks
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
paths:
|
| 6 |
+
- '.github/workflows/checks.yml'
|
| 7 |
+
- 'vons/**'
|
| 8 |
+
- 'tools/**'
|
| 9 |
+
- 'tests/**'
|
| 10 |
+
- 'configs/**'
|
| 11 |
+
- 'sdk/**'
|
| 12 |
+
- 'pyproject.toml'
|
| 13 |
+
- '.gitignore'
|
| 14 |
+
pull_request:
|
| 15 |
+
paths:
|
| 16 |
+
- '.github/workflows/checks.yml'
|
| 17 |
+
- 'vons/**'
|
| 18 |
+
- 'tools/**'
|
| 19 |
+
- 'tests/**'
|
| 20 |
+
- 'configs/**'
|
| 21 |
+
- 'sdk/**'
|
| 22 |
+
- 'pyproject.toml'
|
| 23 |
+
- '.gitignore'
|
| 24 |
+
workflow_dispatch:
|
| 25 |
+
|
| 26 |
+
concurrency:
|
| 27 |
+
group: source-checks-${{ github.workflow }}-${{ github.ref }}
|
| 28 |
+
cancel-in-progress: true
|
| 29 |
+
|
| 30 |
+
permissions:
|
| 31 |
+
contents: read
|
| 32 |
+
|
| 33 |
+
jobs:
|
| 34 |
+
python:
|
| 35 |
+
runs-on: ubuntu-latest
|
| 36 |
+
steps:
|
| 37 |
+
- uses: actions/checkout@v4
|
| 38 |
+
- uses: actions/setup-python@v5
|
| 39 |
+
with:
|
| 40 |
+
python-version: '3.11'
|
| 41 |
+
- run: python -m pip install -e '.[dev]' numpy onnx onnxruntime
|
| 42 |
+
- run: python tools/prepare_public_release.py
|
| 43 |
+
- run: ruff check .
|
| 44 |
+
- run: python -m pytest -q
|
| 45 |
+
typescript:
|
| 46 |
+
runs-on: ubuntu-latest
|
| 47 |
+
defaults:
|
| 48 |
+
run:
|
| 49 |
+
working-directory: sdk/typescript
|
| 50 |
+
steps:
|
| 51 |
+
- uses: actions/checkout@v4
|
| 52 |
+
- uses: actions/setup-node@v4
|
| 53 |
+
with:
|
| 54 |
+
node-version: '22'
|
| 55 |
+
cache: npm
|
| 56 |
+
cache-dependency-path: sdk/typescript/package-lock.json
|
| 57 |
+
- run: npm ci
|
| 58 |
+
- run: npm test
|
| 59 |
+
- run: npm run build:browser
|
| 60 |
+
- run: npm run build:benchmark
|
| 61 |
+
- run: npm run build:parity
|
.gitignore
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Public source allowlist. New root paths stay local until explicitly reviewed.
|
| 2 |
+
/*
|
| 3 |
+
!/.gitignore
|
| 4 |
+
!/.gitattributes
|
| 5 |
+
!/.github/
|
| 6 |
+
!/README.md
|
| 7 |
+
!/CONTRIBUTING.md
|
| 8 |
+
!/CITATION.cff
|
| 9 |
+
!/LICENSE
|
| 10 |
+
!/NOTICE
|
| 11 |
+
!/pyproject.toml
|
| 12 |
+
!/configs/
|
| 13 |
+
!/docs/
|
| 14 |
+
!/sdk/
|
| 15 |
+
!/tests/
|
| 16 |
+
!/tools/
|
| 17 |
+
!/vons/
|
| 18 |
+
|
| 19 |
+
# Internal planning, historical claims and agent/session records stay local.
|
| 20 |
+
/docs/*
|
| 21 |
+
!/docs/KASI_INTEGRATION.md
|
| 22 |
+
!/docs/MODEL_CARD.md
|
| 23 |
+
!/docs/PUBLIC_RELEASE.md
|
| 24 |
+
!/docs/HUGGING_FACE.md
|
| 25 |
+
!/docs/LICENSING.md
|
| 26 |
+
!/docs/COMMUNITY.md
|
| 27 |
+
!/docs/HUB_PUBLISHING.md
|
| 28 |
+
!/docs/publications/
|
| 29 |
+
!/docs/assets/
|
| 30 |
+
/docs/publications/*
|
| 31 |
+
!/docs/publications/README.md
|
| 32 |
+
!/docs/publications/LICENSE.md
|
| 33 |
+
!/docs/publications/ARXIV.md
|
| 34 |
+
/docs/assets/*
|
| 35 |
+
!/docs/assets/inlevel9-signature.png
|
| 36 |
+
!/docs/assets/inlevel9-signature-dark.png
|
| 37 |
+
/configs/*
|
| 38 |
+
!/configs/pilot.json
|
| 39 |
+
!/configs/pilot-diffusion.json
|
| 40 |
+
!/configs/public-release.json
|
| 41 |
+
/sdk/*
|
| 42 |
+
!/sdk/typescript/
|
| 43 |
+
/sdk/typescript/*
|
| 44 |
+
!/sdk/typescript/README.md
|
| 45 |
+
!/sdk/typescript/LICENSE
|
| 46 |
+
!/sdk/typescript/package.json
|
| 47 |
+
!/sdk/typescript/package-lock.json
|
| 48 |
+
!/sdk/typescript/src/
|
| 49 |
+
!/sdk/typescript/test/
|
| 50 |
+
!/sdk/typescript/demo/
|
| 51 |
+
|
| 52 |
+
# Raw/generated data, weights, evidence packets and local publications.
|
| 53 |
+
data/
|
| 54 |
+
artifacts/
|
| 55 |
+
reports/
|
| 56 |
+
research/
|
| 57 |
+
models/
|
| 58 |
+
weights/
|
| 59 |
+
checkpoints/
|
| 60 |
+
outputs/
|
| 61 |
+
releases/
|
| 62 |
+
private/
|
| 63 |
+
restricted/
|
| 64 |
+
*.onnx
|
| 65 |
+
*.onnx.*
|
| 66 |
+
*.safetensors
|
| 67 |
+
*.gguf
|
| 68 |
+
*.ggml
|
| 69 |
+
*.pt
|
| 70 |
+
*.pth
|
| 71 |
+
*.ckpt
|
| 72 |
+
*.bin
|
| 73 |
+
*.h5
|
| 74 |
+
*.hdf5
|
| 75 |
+
*.npz
|
| 76 |
+
*.npy
|
| 77 |
+
*.pkl
|
| 78 |
+
*.pickle
|
| 79 |
+
*.parquet
|
| 80 |
+
*.arrow
|
| 81 |
+
*.jsonl
|
| 82 |
+
*.zip
|
| 83 |
+
*.tar
|
| 84 |
+
*.tar.*
|
| 85 |
+
*.tgz
|
| 86 |
+
*.7z
|
| 87 |
+
*.dmg
|
| 88 |
+
*.pdf
|
| 89 |
+
# Only the exact reviewed publication bytes are eligible for staging.
|
| 90 |
+
!/docs/publications/vons-paper-v1.0.pdf
|
| 91 |
+
!/docs/publications/vons-tech-report-v1.0.1.pdf
|
| 92 |
+
|
| 93 |
+
# Credentials, machine-specific config, private prompts and service caches.
|
| 94 |
+
.env
|
| 95 |
+
.env.*
|
| 96 |
+
*.env
|
| 97 |
+
*.pem
|
| 98 |
+
*.key
|
| 99 |
+
*.p12
|
| 100 |
+
*.pfx
|
| 101 |
+
*.jks
|
| 102 |
+
credentials*.json
|
| 103 |
+
secrets*.json
|
| 104 |
+
service-account*.json
|
| 105 |
+
token.json
|
| 106 |
+
.npmrc
|
| 107 |
+
.pypirc
|
| 108 |
+
.netrc
|
| 109 |
+
.aws/
|
| 110 |
+
.ssh/
|
| 111 |
+
.config/
|
| 112 |
+
.cache/
|
| 113 |
+
.huggingface/
|
| 114 |
+
.claude/
|
| 115 |
+
.codex/
|
| 116 |
+
.cursor/
|
| 117 |
+
.trae/
|
| 118 |
+
.idea/
|
| 119 |
+
.vscode/
|
| 120 |
+
*.local.*
|
| 121 |
+
*.log
|
| 122 |
+
*.sqlite*
|
| 123 |
+
*.db
|
| 124 |
+
*.bak
|
| 125 |
+
*.swp
|
| 126 |
+
*~
|
| 127 |
+
.git_backup/
|
| 128 |
+
|
| 129 |
+
# Environments, dependencies and generated browser bundles.
|
| 130 |
+
.venv/
|
| 131 |
+
venv/
|
| 132 |
+
node_modules/
|
| 133 |
+
__pycache__/
|
| 134 |
+
*.py[cod]
|
| 135 |
+
.pytest_cache/
|
| 136 |
+
.mypy_cache/
|
| 137 |
+
.ruff_cache/
|
| 138 |
+
.tox/
|
| 139 |
+
.coverage*
|
| 140 |
+
htmlcov/
|
| 141 |
+
coverage/
|
| 142 |
+
build/
|
| 143 |
+
dist/
|
| 144 |
+
*.egg-info/
|
| 145 |
+
/sdk/typescript/demo/*.js
|
| 146 |
+
/sdk/typescript/demo/*.js.map
|
| 147 |
+
.DS_Store
|
| 148 |
+
Thumbs.db
|
CITATION.cff
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cff-version: 1.2.0
|
| 2 |
+
message: "If you use this research source, cite this version and the accompanying publications when available."
|
| 3 |
+
type: software
|
| 4 |
+
title: "Vons: Compact Local Decision Models for Frontier-Agent Workflows"
|
| 5 |
+
version: 0.1.0
|
| 6 |
+
authors:
|
| 7 |
+
- family-names: Ahn
|
| 8 |
+
given-names: Kwangseob
|
| 9 |
+
affiliation: "INLEVEL9 / SEJONG UNIV."
|
| 10 |
+
email: oswarld@inlevel9.com
|
| 11 |
+
abstract: "Research source for bounded local candidate selection and abstention, with host-owned execution and consent."
|
CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Contributing to Vons
|
| 2 |
+
|
| 3 |
+
Help make bounded local decisions easier to understand, reproduce and improve.
|
| 4 |
+
Questions, unsuccessful trials, documentation fixes and small code changes are
|
| 5 |
+
welcome. Start with the [community guide](docs/COMMUNITY.md).
|
| 6 |
+
|
| 7 |
+
## Choose a contribution
|
| 8 |
+
|
| 9 |
+
- Share an experience using the **Experience report** issue form. Describe what
|
| 10 |
+
happened, including failures; a positive result is not required.
|
| 11 |
+
- Submit a **Reproduction report** with exact commands, versions, denominators
|
| 12 |
+
and the scope of the claim you checked.
|
| 13 |
+
- Use **Bug report** for a minimal, preferably synthetic, reproduction.
|
| 14 |
+
- Propose a focused change through a pull request. Discuss changes to the public
|
| 15 |
+
contract, data handling, model architecture or dependencies before implementing
|
| 16 |
+
them. Small documentation corrections can go directly to a pull request.
|
| 17 |
+
|
| 18 |
+
The intended repository is [inlevel9-com/Jons](https://github.com/inlevel9-com/Jons).
|
| 19 |
+
Its URL spelling differs from the Vons project name. It was private and empty
|
| 20 |
+
when these instructions were prepared; public participation starts after the
|
| 21 |
+
maintainer publishes the reviewed source and verifies the issue forms.
|
| 22 |
+
|
| 23 |
+
## Work locally
|
| 24 |
+
|
| 25 |
+
Use Python 3.10 or later. From a source checkout:
|
| 26 |
+
|
| 27 |
+
```sh
|
| 28 |
+
python3 -m venv .venv
|
| 29 |
+
. .venv/bin/activate
|
| 30 |
+
python -m pip install -e '.[dev]'
|
| 31 |
+
python -m pytest -q
|
| 32 |
+
ruff check .
|
| 33 |
+
python tools/prepare_public_release.py
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
On Windows, activate with `.venv\Scripts\Activate.ps1` in PowerShell. Tests for
|
| 37 |
+
optional training/export dependencies may skip explicitly. Do not download
|
| 38 |
+
restricted data or model weights simply to remove a skip. For TypeScript work,
|
| 39 |
+
use the [SDK checks](sdk/typescript/README.md).
|
| 40 |
+
|
| 41 |
+
Keep one purpose per change. Add behavior-focused regression coverage for fixes.
|
| 42 |
+
Report checks you ran, checks you could not run, and the reason. English is used
|
| 43 |
+
for code, comments and repository documentation; community questions and
|
| 44 |
+
experience reports may be in English or Korean.
|
| 45 |
+
|
| 46 |
+
## Preserve the boundaries
|
| 47 |
+
|
| 48 |
+
The general decision contract and the KASI compatibility adapter are separate.
|
| 49 |
+
Model output never executes a tool, grants consent or authorizes a high-risk
|
| 50 |
+
action. Self-reported confidence is not a calibrated probability. Missing
|
| 51 |
+
measurements stay unavailable; design targets are not achieved results.
|
| 52 |
+
|
| 53 |
+
Use synthetic or redacted examples in public reports. Do not attach credentials,
|
| 54 |
+
private prompts, customer traces, personal data, restricted benchmark rows,
|
| 55 |
+
model weights or private evidence archives. Keep your own raw records locally;
|
| 56 |
+
share only material you are entitled to disclose. Contact
|
| 57 |
+
[oswarld@inlevel9.com](mailto:oswarld@inlevel9.com) before disclosing a security
|
| 58 |
+
issue; the first message should describe the issue without credentials or
|
| 59 |
+
sensitive attachments.
|
| 60 |
+
|
| 61 |
+
New public files must be reviewed and added to `configs/public-release.json`
|
| 62 |
+
and, where necessary, `.gitignore`. Never force-add ignored research material.
|
| 63 |
+
|
| 64 |
+
## Review and attribution
|
| 65 |
+
|
| 66 |
+
Maintainers review scope, correctness, evidence and redistribution rights before
|
| 67 |
+
merging. An issue, testimonial or model-assisted review is not acceptance,
|
| 68 |
+
independent replication or peer review. Review may request changes or decline a
|
| 69 |
+
proposal; no response-time or merge guarantee is implied.
|
| 70 |
+
|
| 71 |
+
Be specific and respectful. Critique claims and code rather than people. Spam,
|
| 72 |
+
harassment, fabricated evidence and exposed private information may be removed;
|
| 73 |
+
technical criticism and negative results are welcome. Corrections should keep
|
| 74 |
+
the original claim and explain what changed whenever it is safe to do so.
|
| 75 |
+
|
| 76 |
+
The software uses the [Vons Community and Commercial License](LICENSE), with
|
| 77 |
+
separate [article terms](docs/publications/LICENSE.md). Submitting an issue does
|
| 78 |
+
not grant software-use rights or permission to use it as a promotional quote.
|
| 79 |
+
Code contributions are reviewed for compatibility with the repository license;
|
| 80 |
+
identify third-party material and obtain any required employer permission.
|
| 81 |
+
No copyright assignment or separate contributor agreement is created here.
|
LICENSE
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Vons Community and Commercial License 1.0
|
| 2 |
+
SPDX-License-Identifier: LicenseRef-Vons-Community-Commercial-1.0
|
| 3 |
+
|
| 4 |
+
1. Scope and definitions
|
| 5 |
+
|
| 6 |
+
"Licensor" means each copyright holder who makes material available under this
|
| 7 |
+
license. "Software" means the Vons-owned source code, configuration, tests and
|
| 8 |
+
supporting software documentation accompanied by this license, including copies
|
| 9 |
+
and modifications. "You" means the individual or legal entity exercising the
|
| 10 |
+
permissions below, including persons acting on that entity's behalf.
|
| 11 |
+
|
| 12 |
+
This license does not cover material with a separate license notice. In
|
| 13 |
+
particular, the paper and Tech Report and their original research figures are
|
| 14 |
+
licensed separately under CC BY 4.0. Third-party dependencies, upstream model
|
| 15 |
+
assets, datasets, trademarks and logos are not relicensed by this document.
|
| 16 |
+
No model weights or benchmark datasets are supplied with this source release.
|
| 17 |
+
|
| 18 |
+
"Noncommercial Use" means personal learning, independent noncommercial
|
| 19 |
+
research, or noncommercial education, research or public-benefit activity by a
|
| 20 |
+
nonprofit or educational institution, where the activity is not undertaken for
|
| 21 |
+
commercial advantage, monetary compensation, or on behalf of a business.
|
| 22 |
+
Nonprofit or educational status alone does not make commercial activity
|
| 23 |
+
Noncommercial Use. Unrestricted donations that do not buy a service or benefit
|
| 24 |
+
do not by themselves make an otherwise qualifying activity commercial.
|
| 25 |
+
|
| 26 |
+
"Commercial Use" includes paid consulting or client work; use in or to support
|
| 27 |
+
a monetized product, service, SaaS, API, hosted or managed service; internal
|
| 28 |
+
business operations; and evaluation, research or deployment by, for, or on
|
| 29 |
+
behalf of a for-profit enterprise. Contractors and affiliates acting for such
|
| 30 |
+
an enterprise require the same permission. Personal use by an employee outside
|
| 31 |
+
their employment, without business benefit, may qualify as Noncommercial Use.
|
| 32 |
+
|
| 33 |
+
2. Free community permission
|
| 34 |
+
|
| 35 |
+
Subject to this license, each Licensor grants You a worldwide, nonexclusive,
|
| 36 |
+
royalty-free copyright license to use, reproduce, study, modify, display and
|
| 37 |
+
distribute the Software solely for Noncommercial Use. You may provide access
|
| 38 |
+
to an otherwise qualifying noncommercial service under these conditions.
|
| 39 |
+
Only rights that a Licensor is entitled to grant are included.
|
| 40 |
+
|
| 41 |
+
3. Redistribution and attribution
|
| 42 |
+
|
| 43 |
+
When distributing copies or substantial portions of the Software, including
|
| 44 |
+
modified copies or a compiled form, You must:
|
| 45 |
+
|
| 46 |
+
(a) retain this license, copyright notices and attribution notices;
|
| 47 |
+
(b) clearly identify Your modifications and preserve their provenance;
|
| 48 |
+
(c) identify the Vons version or source revision used where available; and
|
| 49 |
+
(d) convey these same use restrictions for the covered Software, without
|
| 50 |
+
representing that You can grant Commercial Use rights.
|
| 51 |
+
|
| 52 |
+
Do not remove notices, misrepresent origin, or imply endorsement by a Licensor,
|
| 53 |
+
INLEVEL9, Sejong University or any other named institution. For research that
|
| 54 |
+
uses Vons, include an appropriate citation to the software and relevant paper
|
| 55 |
+
or Tech Report when reporting results. An unavailable DOI is not required.
|
| 56 |
+
Attribution or citation does not authorize an otherwise restricted use.
|
| 57 |
+
|
| 58 |
+
4. Enterprise and commercial permission
|
| 59 |
+
|
| 60 |
+
Commercial Use is not granted by this license. It requires a separate written
|
| 61 |
+
commercial agreement from the relevant Licensor before the activity begins.
|
| 62 |
+
There is no automatic enterprise evaluation or free-trial exception. Contact
|
| 63 |
+
oswarld@inlevel9.com to arrange the scope, fees and permitted entities.
|
| 64 |
+
This license sets no price, billing unit, service level or support commitment.
|
| 65 |
+
A commercial agreement applies only to the parties and scope it identifies.
|
| 66 |
+
|
| 67 |
+
5. Outputs and independent rights
|
| 68 |
+
|
| 69 |
+
This license claims no ownership or royalty over Your inference outputs merely
|
| 70 |
+
because they were produced using the Software. Outputs do not authorize an
|
| 71 |
+
unlicensed deployment and may contain material subject to independent rights.
|
| 72 |
+
Your inputs remain subject to their own rights and permissions.
|
| 73 |
+
|
| 74 |
+
No trademark, logo, certification or endorsement rights are granted, except
|
| 75 |
+
the descriptive use needed to comply with the notices above. No patent license
|
| 76 |
+
is granted by this document. Statutory exceptions and other independently
|
| 77 |
+
obtained permissions are unaffected. This license does not revoke rights
|
| 78 |
+
previously granted under another license for an earlier copy, and does not
|
| 79 |
+
restrict separately licensed third-party components.
|
| 80 |
+
|
| 81 |
+
6. Termination and cure
|
| 82 |
+
|
| 83 |
+
Your permissions terminate automatically if You materially breach this
|
| 84 |
+
license. They are reinstated when the breach is cured within 30 days after
|
| 85 |
+
You first discover it, provided this is Your first breach. Otherwise written
|
| 86 |
+
reinstatement from the relevant Licensor is required. Termination does not
|
| 87 |
+
affect compliant recipients' direct permissions or obligations that by their
|
| 88 |
+
nature survive, including the notices and disclaimers below.
|
| 89 |
+
|
| 90 |
+
7. Warranty disclaimer and liability
|
| 91 |
+
|
| 92 |
+
TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE SOFTWARE IS PROVIDED
|
| 93 |
+
"AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WARRANTIES
|
| 94 |
+
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND
|
| 95 |
+
NONINFRINGEMENT. THE LICENSORS ARE NOT LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 96 |
+
LIABILITY ARISING FROM THE SOFTWARE OR ITS USE, WHETHER IN CONTRACT, TORT OR
|
| 97 |
+
OTHERWISE. NOTHING HERE EXCLUDES LIABILITY THAT CANNOT LAWFULLY BE EXCLUDED.
|
| 98 |
+
|
| 99 |
+
8. General
|
| 100 |
+
|
| 101 |
+
Rights not expressly granted are reserved. No later license version applies
|
| 102 |
+
automatically. If a provision is unenforceable, the remaining provisions remain
|
| 103 |
+
in effect to the extent permitted by law. This is a source-available license
|
| 104 |
+
with use restrictions; it is not an OSI-approved open-source license.
|
README.md
CHANGED
|
@@ -1,3 +1,93 @@
|
|
| 1 |
---
|
| 2 |
-
license:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
license: other
|
| 3 |
+
license_name: vons-community-commercial-1.0
|
| 4 |
+
license_link: LICENSE
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
tags:
|
| 8 |
+
- vons
|
| 9 |
+
- research
|
| 10 |
+
- candidate-selection
|
| 11 |
+
- onnx
|
| 12 |
---
|
| 13 |
+
|
| 14 |
+

|
| 15 |
+
|
| 16 |
+
# Vons
|
| 17 |
+
|
| 18 |
+
**Plan globally. Decide locally. Keep the host in control.**
|
| 19 |
+
|
| 20 |
+
Compact local decision models for frontier-agent workflows.
|
| 21 |
+
|
| 22 |
+
**Kwangseob Ahn** · **INLEVEL9 / SEJONG UNIV.** · **oswarld@inlevel9.com**
|
| 23 |
+
|
| 24 |
+
## Source preview
|
| 25 |
+
|
| 26 |
+
This repository contains research source and documentation. It does not contain
|
| 27 |
+
pretrained model weights, tokenizer assets or datasets. Hosted inference and
|
| 28 |
+
`from_pretrained` loading are not available from this source preview.
|
| 29 |
+
|
| 30 |
+
Vons studies Direct and Diffusion candidate scoring with abstention. The host
|
| 31 |
+
owns execution, permission and consent. Model output never authorizes an action.
|
| 32 |
+
English-first operation and a model asset budget under 64 MiB are design targets;
|
| 33 |
+
they are not general quality or complete browser-package guarantees.
|
| 34 |
+
|
| 35 |
+
See [the model card](docs/MODEL_CARD.md),
|
| 36 |
+
[the TypeScript SDK](sdk/typescript/README.md),
|
| 37 |
+
[the paper and Tech Report](docs/publications/README.md), and
|
| 38 |
+
[release preparation](docs/PUBLIC_RELEASE.md).
|
| 39 |
+
|
| 40 |
+
## Read, try and contribute
|
| 41 |
+
|
| 42 |
+
This source preview is distributed through [INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons).
|
| 43 |
+
Source and contribution reports belong in
|
| 44 |
+
[inlevel9-com/Vons](https://github.com/inlevel9-com/Vons). The reviewed paper and Tech Report are included as PDF downloads below. The software remains a 0.1.0
|
| 45 |
+
research preview. No pretrained weights or working Space are announced.
|
| 46 |
+
|
| 47 |
+
Follow the [community guide](docs/COMMUNITY.md) for the no-weight walkthrough,
|
| 48 |
+
experience and reproduction reports, and the
|
| 49 |
+
[contribution guide](CONTRIBUTING.md) for review expectations. The paper is
|
| 50 |
+
being prepared for [arXiv](docs/publications/ARXIV.md). User reports are not
|
| 51 |
+
independent research validation.
|
| 52 |
+
|
| 53 |
+
Maintainers can use the [Hub upload guide](docs/HUB_PUBLISHING.md) to prepare
|
| 54 |
+
an organization card, source preview and later a separately validated Space.
|
| 55 |
+
|
| 56 |
+
## Paper and Tech Report
|
| 57 |
+
|
| 58 |
+
| Publication | Download |
|
| 59 |
+
| --- | --- |
|
| 60 |
+
| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows — paper v1.0, final editorial revision | [PDF · 15 pages](docs/publications/vons-paper-v1.0.pdf) |
|
| 61 |
+
| Vons: Compact Decision Models for Frontier-Agent Workflows — Technical Report v1.0.1 | [PDF · 20 pages](docs/publications/vons-tech-report-v1.0.1.pdf) |
|
| 62 |
+
|
| 63 |
+
Articles and original figures use **CC BY 4.0**. Their version, SHA-256 digests
|
| 64 |
+
and verification scope are in the [publication index](docs/publications/README.md).
|
| 65 |
+
These preprints preserve negative results and missing measurements. No arXiv
|
| 66 |
+
identifier or peer-review acceptance is announced.
|
| 67 |
+
|
| 68 |
+
## Local use
|
| 69 |
+
|
| 70 |
+
```sh
|
| 71 |
+
python3 -m venv .venv
|
| 72 |
+
. .venv/bin/activate
|
| 73 |
+
python -m pip install -e '.[dev]'
|
| 74 |
+
python -m vons.cli generate-smoke --output data/generated/smoke.jsonl
|
| 75 |
+
python -m vons.cli validate-data --input data/generated/smoke.jsonl
|
| 76 |
+
```
|
| 77 |
+
|
| 78 |
+
The source export includes the research tools and pinned pilot configurations.
|
| 79 |
+
Obtain and review upstream assets separately before running model experiments.
|
| 80 |
+
Raw restricted benchmark data and private evidence are not redistributed here.
|
| 81 |
+
|
| 82 |
+
## Terms
|
| 83 |
+
|
| 84 |
+
Source is distributed under [Vons Community and Commercial License 1.0](LICENSE):
|
| 85 |
+
qualifying noncommercial use is free; enterprise and other commercial use require
|
| 86 |
+
a separate written agreement. Contact oswarld@inlevel9.com for commercial terms.
|
| 87 |
+
This is source-available software, not OSI-approved open source.
|
| 88 |
+
|
| 89 |
+
The paper and Tech Report are distributed under [CC BY 4.0](docs/publications/LICENSE.md).
|
| 90 |
+
This permits commercial article reuse with attribution; software rights are
|
| 91 |
+
separate. Third-party assets retain their own terms. See
|
| 92 |
+
[licensing scope](docs/LICENSING.md). This source release does not establish research validation or a commercial
|
| 93 |
+
customer agreement.
|
configs/pilot-diffusion.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_id": "vons-pilot-diffusion",
|
| 3 |
+
"backend": "diffusion",
|
| 4 |
+
"option_count": 32,
|
| 5 |
+
"encoder": {"name": "google/bert_uncased_L-4_H-256_A-4", "revision": "387825ce42dbb39b87911cdf8e383ee3b25184f8", "max_tokens": 512},
|
| 6 |
+
"data": {"path": "data/generated/pilot.jsonl", "manifest": "artifacts/pilot-diffusion/data-manifest.json"},
|
| 7 |
+
"seed": 7,
|
| 8 |
+
"epochs": 3,
|
| 9 |
+
"batch_size": 16,
|
| 10 |
+
"learning_rate": 0.00005,
|
| 11 |
+
"diffusion_steps": 32,
|
| 12 |
+
"inference_steps": 4,
|
| 13 |
+
"device": "auto",
|
| 14 |
+
"artifact_dir": "artifacts/pilot-diffusion",
|
| 15 |
+
"target_bundle_bytes": 67108864
|
| 16 |
+
}
|
configs/pilot.json
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_id": "vons-pilot-direct",
|
| 3 |
+
"backend": "direct",
|
| 4 |
+
"option_count": 32,
|
| 5 |
+
"encoder": {"name": "google/bert_uncased_L-4_H-256_A-4", "revision": "387825ce42dbb39b87911cdf8e383ee3b25184f8", "max_tokens": 512},
|
| 6 |
+
"data": {"path": "data/generated/pilot.jsonl", "manifest": "artifacts/pilot/data-manifest.json"},
|
| 7 |
+
"seed": 7,
|
| 8 |
+
"epochs": 3,
|
| 9 |
+
"batch_size": 16,
|
| 10 |
+
"learning_rate": 0.00005,
|
| 11 |
+
"diffusion_steps": 32,
|
| 12 |
+
"inference_steps": 4,
|
| 13 |
+
"device": "auto",
|
| 14 |
+
"target_bundle_bytes": 67108864
|
| 15 |
+
}
|
configs/public-release.json
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": "vons.public-source-spec/v1",
|
| 3 |
+
"license_status": "owner_selected_custom_terms_prepared",
|
| 4 |
+
"files": [
|
| 5 |
+
".gitattributes",
|
| 6 |
+
".github/ISSUE_TEMPLATE/bug.yml",
|
| 7 |
+
".github/ISSUE_TEMPLATE/experience.yml",
|
| 8 |
+
".github/ISSUE_TEMPLATE/reproduction.yml",
|
| 9 |
+
".github/PULL_REQUEST_TEMPLATE.md",
|
| 10 |
+
".github/workflows/checks.yml",
|
| 11 |
+
".gitignore",
|
| 12 |
+
"CITATION.cff",
|
| 13 |
+
"CONTRIBUTING.md",
|
| 14 |
+
"LICENSE",
|
| 15 |
+
"README.md",
|
| 16 |
+
"configs/pilot-diffusion.json",
|
| 17 |
+
"configs/pilot.json",
|
| 18 |
+
"configs/public-release.json",
|
| 19 |
+
"docs/COMMUNITY.md",
|
| 20 |
+
"docs/HUB_PUBLISHING.md",
|
| 21 |
+
"docs/HUGGING_FACE.md",
|
| 22 |
+
"docs/KASI_INTEGRATION.md",
|
| 23 |
+
"docs/LICENSING.md",
|
| 24 |
+
"docs/MODEL_CARD.md",
|
| 25 |
+
"docs/PUBLIC_RELEASE.md",
|
| 26 |
+
"docs/assets/inlevel9-signature-dark.png",
|
| 27 |
+
"docs/assets/inlevel9-signature.png",
|
| 28 |
+
"docs/publications/ARXIV.md",
|
| 29 |
+
"docs/publications/LICENSE.md",
|
| 30 |
+
"docs/publications/README.md",
|
| 31 |
+
"docs/publications/vons-paper-v1.0.pdf",
|
| 32 |
+
"docs/publications/vons-tech-report-v1.0.1.pdf",
|
| 33 |
+
"pyproject.toml",
|
| 34 |
+
"sdk/typescript/LICENSE",
|
| 35 |
+
"sdk/typescript/README.md",
|
| 36 |
+
"sdk/typescript/demo/benchmark.html",
|
| 37 |
+
"sdk/typescript/demo/benchmark.ts",
|
| 38 |
+
"sdk/typescript/demo/index.html",
|
| 39 |
+
"sdk/typescript/demo/main.ts",
|
| 40 |
+
"sdk/typescript/demo/parity.html",
|
| 41 |
+
"sdk/typescript/demo/parity.ts",
|
| 42 |
+
"sdk/typescript/package-lock.json",
|
| 43 |
+
"sdk/typescript/package.json",
|
| 44 |
+
"sdk/typescript/src/index.ts",
|
| 45 |
+
"sdk/typescript/src/onnx.ts",
|
| 46 |
+
"sdk/typescript/test/index.test.mjs",
|
| 47 |
+
"sdk/typescript/test/onnx.test.mjs",
|
| 48 |
+
"sdk/typescript/test/parity.test.mjs",
|
| 49 |
+
"tests/test_browser_benchmark.py",
|
| 50 |
+
"tests/test_browser_benchmark_summary.py",
|
| 51 |
+
"tests/test_browser_parity_reference.py",
|
| 52 |
+
"tests/test_bundle.py",
|
| 53 |
+
"tests/test_calibration.py",
|
| 54 |
+
"tests/test_contract.py",
|
| 55 |
+
"tests/test_data.py",
|
| 56 |
+
"tests/test_evaluation.py",
|
| 57 |
+
"tests/test_merge_browser_benchmarks.py",
|
| 58 |
+
"tests/test_mind2web.py",
|
| 59 |
+
"tests/test_mind2web_import.py",
|
| 60 |
+
"tests/test_mind2web_manifest.py",
|
| 61 |
+
"tests/test_mind2web_prediction.py",
|
| 62 |
+
"tests/test_mind2web_replay.py",
|
| 63 |
+
"tests/test_mind2web_runner.py",
|
| 64 |
+
"tests/test_public_release.py",
|
| 65 |
+
"tests/test_quality_summary.py",
|
| 66 |
+
"tests/test_regression.py",
|
| 67 |
+
"tests/test_shape_benchmark.py",
|
| 68 |
+
"tests/test_shared_bundle.py",
|
| 69 |
+
"tools/archive_manifest.py",
|
| 70 |
+
"tools/audit_local_model_alignment.py",
|
| 71 |
+
"tools/benchmark_onnx_bundle.py",
|
| 72 |
+
"tools/benchmark_onnx_shapes.py",
|
| 73 |
+
"tools/build_claim_ledger.py",
|
| 74 |
+
"tools/build_mind2web_manifest.py",
|
| 75 |
+
"tools/evaluate_mind2web.py",
|
| 76 |
+
"tools/export_shared_bundle.py",
|
| 77 |
+
"tools/freeze_publication_packet.py",
|
| 78 |
+
"tools/import_mind2web.py",
|
| 79 |
+
"tools/merge_browser_benchmarks.py",
|
| 80 |
+
"tools/predict_mind2web.py",
|
| 81 |
+
"tools/prepare_public_release.py",
|
| 82 |
+
"tools/recover_historical_manifest.py",
|
| 83 |
+
"tools/replay_mind2web_evaluation.py",
|
| 84 |
+
"tools/summarize_browser_benchmark.py",
|
| 85 |
+
"tools/summarize_quality.py",
|
| 86 |
+
"tools/verify_browser_benchmark.py",
|
| 87 |
+
"tools/verify_browser_parity.py",
|
| 88 |
+
"tools/verify_onnx_bundle.py",
|
| 89 |
+
"vons/__init__.py",
|
| 90 |
+
"vons/bundle.py",
|
| 91 |
+
"vons/calibration.py",
|
| 92 |
+
"vons/cli.py",
|
| 93 |
+
"vons/contract.py",
|
| 94 |
+
"vons/data.py",
|
| 95 |
+
"vons/evaluation.py",
|
| 96 |
+
"vons/kasi.py",
|
| 97 |
+
"vons/mind2web.py",
|
| 98 |
+
"vons/models.py",
|
| 99 |
+
"vons/ollama.py",
|
| 100 |
+
"vons/research.py",
|
| 101 |
+
"vons/training.py"
|
| 102 |
+
],
|
| 103 |
+
"reviewed_binaries": {
|
| 104 |
+
"docs/assets/inlevel9-signature.png": "f7514b56b2a93293054b3d22634ac5f0cb11e00a370f53d8d9f934c55a21bbb5",
|
| 105 |
+
"docs/assets/inlevel9-signature-dark.png": "d6645528158842af77a89de97361fa0c521fbc5a3c0ea6e996f9c912cfcfe9b5",
|
| 106 |
+
"docs/publications/vons-paper-v1.0.pdf": "ae0e09c60e17a98b52bb25c611b2489d10b18bd98d87974f2e8dd25d746a5fc8",
|
| 107 |
+
"docs/publications/vons-tech-report-v1.0.1.pdf": "33076d9e52d4a073060066215dd390e3b0bf0cdef48c5ffb2a7c5189854254ed"
|
| 108 |
+
},
|
| 109 |
+
"software_license": "LicenseRef-Vons-Community-Commercial-1.0",
|
| 110 |
+
"article_license": "CC-BY-4.0"
|
| 111 |
+
}
|
docs/COMMUNITY.md
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Read, try, share and contribute
|
| 2 |
+
|
| 3 |
+
Vons explores compact local decisions inside agent workflows. A planner supplies
|
| 4 |
+
state and bounded candidates; the host retains execution, policy and consent.
|
| 5 |
+
The community should help people understand this boundary, try the software and
|
| 6 |
+
report what works or fails.
|
| 7 |
+
|
| 8 |
+
## Start here
|
| 9 |
+
|
| 10 |
+
| Your goal | Entry point | Current availability |
|
| 11 |
+
| --- | --- | --- |
|
| 12 |
+
| Understand Vons | [README](../README.md) and [model card](MODEL_CARD.md) | Included in this source preview |
|
| 13 |
+
| Read the research | [Paper and Tech Report](publications/README.md) | Coming soon; v1.0 manuscripts are in preparation and excluded from this release |
|
| 14 |
+
| Try without weights | The walkthrough below | Deterministic contract and synthetic data only |
|
| 15 |
+
| Run model inference | [TypeScript SDK](../sdk/typescript/README.md) | Requires separately obtained, reviewed model/tokenizer assets |
|
| 16 |
+
| Share experience | [Experience report](https://github.com/inlevel9-com/Vons/issues/new?template=experience.yml) | GitHub account required to submit |
|
| 17 |
+
| Reproduce a result | [Reproduction report](https://github.com/inlevel9-com/Vons/issues/new?template=reproduction.yml) | Form included; evidence reviewed separately |
|
| 18 |
+
| Improve Vons | [Contributing](../CONTRIBUTING.md) | Documentation, tests and scoped code changes welcome |
|
| 19 |
+
|
| 20 |
+
The destinations are [GitHub: inlevel9-com/Vons](https://github.com/inlevel9-com/Vons)
|
| 21 |
+
and [Hugging Face: INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons).
|
| 22 |
+
The source preview includes documentation and GitHub issue forms for feedback.
|
| 23 |
+
The paper and Tech Report will be added after v1.0 review; this release contains
|
| 24 |
+
neither manuscript. A source upload does not provide hosted model inference.
|
| 25 |
+
The current software terms require a separate agreement for
|
| 26 |
+
enterprise/commercial use, including enterprise evaluation. Reading the paper
|
| 27 |
+
under its article terms is separate from software use.
|
| 28 |
+
|
| 29 |
+
## A five-minute first exploration
|
| 30 |
+
|
| 31 |
+
Five minutes is a usability target, not a measured onboarding result. Start in
|
| 32 |
+
the root of a source checkout with Python 3.10+. This path needs no model assets,
|
| 33 |
+
GPU, API token or hosted inference account. Python packaging may need internet
|
| 34 |
+
access to install its build tools.
|
| 35 |
+
|
| 36 |
+
```sh
|
| 37 |
+
python3 -m venv .venv
|
| 38 |
+
. .venv/bin/activate
|
| 39 |
+
python -m pip install -e .
|
| 40 |
+
python -m vons.cli generate-smoke --output data/generated/community-smoke.jsonl
|
| 41 |
+
python -m vons.cli validate-data --input data/generated/community-smoke.jsonl
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
On Windows PowerShell use `.venv\Scripts\Activate.ps1` for activation. The
|
| 45 |
+
generator reports four synthetic rows. Validation prints their data manifest;
|
| 46 |
+
this checks the data contract, not model quality.
|
| 47 |
+
|
| 48 |
+
Run this Python example in the same environment:
|
| 49 |
+
|
| 50 |
+
```python
|
| 51 |
+
from vons import KASIAdapter, KASIProposal
|
| 52 |
+
|
| 53 |
+
adapter = KASIAdapter(tool_policy={"read_weather": "low"})
|
| 54 |
+
for tool in ("read_weather", "unknown_tool"):
|
| 55 |
+
proposal = KASIProposal.from_mapping({
|
| 56 |
+
"calls": [{"name": tool}],
|
| 57 |
+
"confidence": 0.9,
|
| 58 |
+
"risk": "low",
|
| 59 |
+
})
|
| 60 |
+
decision = adapter.decide(proposal)
|
| 61 |
+
print(tool, decision.action.value, decision.reason)
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
Expected output:
|
| 65 |
+
|
| 66 |
+
```text
|
| 67 |
+
read_weather call approved_by_host_policy
|
| 68 |
+
unknown_tool refuse unregistered_or_no_proposed_call
|
| 69 |
+
```
|
| 70 |
+
|
| 71 |
+
This is the deterministic KASI adapter, not a learned Vons prediction. The word
|
| 72 |
+
`call` is a returned decision; the example never fetches weather or executes a
|
| 73 |
+
tool. The host supplies the tool policy. The literal confidence is sample input,
|
| 74 |
+
not a calibrated probability. See the [general scope](MODEL_CARD.md) and
|
| 75 |
+
[KASI integration](KASI_INTEGRATION.md) before building an integration.
|
| 76 |
+
|
| 77 |
+
## Share an honest experience
|
| 78 |
+
|
| 79 |
+
Use a report form with the source revision or release-manifest digest, scenario,
|
| 80 |
+
OS/device, runtime/provider, exact command, expected behavior and observed
|
| 81 |
+
behavior. Mark whether you tried the contract, a synthetic example or real model
|
| 82 |
+
inference. Include attempted, successful, abstained and failed counts with
|
| 83 |
+
definitions; explain any overlapping categories. Report unavailable quantities
|
| 84 |
+
as unavailable and keep errors in the denominator.
|
| 85 |
+
|
| 86 |
+
A short negative report is useful. Personal experience and measured benchmarks
|
| 87 |
+
serve different purposes: user ratings, stars and downloads do not establish
|
| 88 |
+
accuracy, safety or independent replication. Public reports are not automatically
|
| 89 |
+
training data or promotional testimonials. Link only safe evidence you have
|
| 90 |
+
reviewed and have permission to share. No name, employer or private workflow is
|
| 91 |
+
required beyond the hosting service's account requirements.
|
| 92 |
+
|
| 93 |
+
## A practical platform roadmap
|
| 94 |
+
|
| 95 |
+
These are proposed acceptance criteria, not completed or measured milestones.
|
| 96 |
+
|
| 97 |
+
1. **Reading and participation:** publish a reviewed source preview, readable
|
| 98 |
+
research summary and accurate availability notices. A signed-out visitor can
|
| 99 |
+
read the documentation; a signed-in contributor can submit each issue form
|
| 100 |
+
and find the contribution rules. Check keyboard navigation and narrow screens.
|
| 101 |
+
2. **First use:** test the no-weight walkthrough from a clean checkout. Invite
|
| 102 |
+
voluntary reports of completion, failure and time spent. Keep counts and
|
| 103 |
+
denominators; do not infer actual use from page views or stars.
|
| 104 |
+
3. **Real inference:** offer a versioned demo only after permitted model assets,
|
| 105 |
+
tokenizer, notices, provider evidence and numerical parity are reviewed.
|
| 106 |
+
Clearly label unsupported requests and known failures. Host policy still
|
| 107 |
+
controls actions. Do not present static or simulated results as inference.
|
| 108 |
+
4. **Community learning:** triage reports, publish reproducible improvements,
|
| 109 |
+
credit contributions and keep corrections visible. Record submitted reports,
|
| 110 |
+
independently reproduced reports and accepted contributions separately.
|
| 111 |
+
|
| 112 |
+
Start with the existing GitHub and Hugging Face channels. A dedicated website,
|
| 113 |
+
review database, accounts and telemetry need a separate implementation decision;
|
| 114 |
+
none is required for the first source preview. Reading should remain available
|
| 115 |
+
without a Vons account. Follow the [contribution and moderation rules](../CONTRIBUTING.md).
|
docs/HUB_PUBLISHING.md
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Upload the Vons source preview to Hugging Face
|
| 2 |
+
|
| 3 |
+
This is an owner-operated upload guide. The current local preparation contains
|
| 4 |
+
97 reviewed files, including the final paper and Tech Report PDFs. Drafts,
|
| 5 |
+
source manuscript bundles, model weights, restricted data and private records
|
| 6 |
+
are excluded.
|
| 7 |
+
|
| 8 |
+
## Destination and scope
|
| 9 |
+
|
| 10 |
+
Use the existing model repository [INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons).
|
| 11 |
+
The organization page [INLEVEL9](https://huggingface.co/INLEVEL9) is its parent,
|
| 12 |
+
not a folder-upload destination. There is no need to create another repository,
|
| 13 |
+
a dataset or a Space for this source preview.
|
| 14 |
+
|
| 15 |
+
At the last read-only check on 2026-09-25, the Hub repository was public and
|
| 16 |
+
contained only `.gitattributes` and a minimal Apache-2.0 README. The prepared
|
| 17 |
+
root README and LICENSE replace that placeholder with the selected Vons
|
| 18 |
+
Community and Commercial License 1.0. The article CC BY 4.0 terms are separate.
|
| 19 |
+
|
| 20 |
+
A commit to this public repository makes uploaded files publicly readable.
|
| 21 |
+
Uploading source does not create hosted inference, pretrained weights or a
|
| 22 |
+
working `from_pretrained` package. The card states those limits explicitly.
|
| 23 |
+
The main code/contribution destination is
|
| 24 |
+
[inlevel9-com/Vons](https://github.com/inlevel9-com/Vons). Its visibility is
|
| 25 |
+
controlled separately from Hugging Face.
|
| 26 |
+
|
| 27 |
+
## 1. Install and sign in
|
| 28 |
+
|
| 29 |
+
On macOS with Homebrew:
|
| 30 |
+
|
| 31 |
+
```sh
|
| 32 |
+
brew install hf
|
| 33 |
+
hf auth login
|
| 34 |
+
hf auth whoami
|
| 35 |
+
```
|
| 36 |
+
|
| 37 |
+
If Homebrew already installed `hf`, use `brew upgrade hf` when an update is
|
| 38 |
+
needed. Current CLI versions offer browser login: follow the displayed URL and
|
| 39 |
+
code, and sign in using your account with write access to `INLEVEL9/Vons`.
|
| 40 |
+
`hf auth whoami` confirms the account and organization membership; membership
|
| 41 |
+
alone does not guarantee repository write permission.
|
| 42 |
+
|
| 43 |
+
If your CLI asks for a token instead, create a user access token in
|
| 44 |
+
[Hugging Face token settings](https://huggingface.co/settings/tokens), preferably
|
| 45 |
+
scoped to write to `INLEVEL9/Vons`, and enter it at the hidden login prompt.
|
| 46 |
+
Keep credentials out of source files, shell command arguments and issue reports.
|
| 47 |
+
See the official [CLI guide](https://huggingface.co/docs/huggingface_hub/guides/cli).
|
| 48 |
+
|
| 49 |
+
## 2. Use the reviewed export
|
| 50 |
+
|
| 51 |
+
Run the source audit from the Vons checkout:
|
| 52 |
+
|
| 53 |
+
```sh
|
| 54 |
+
python3 tools/prepare_public_release.py
|
| 55 |
+
```
|
| 56 |
+
|
| 57 |
+
The current prepared upload folder is `releases/huggingface-source-v8`.
|
| 58 |
+
It contains the 97 reviewed files plus a local `RELEASE_MANIFEST.json` receipt
|
| 59 |
+
with every file's byte count and SHA-256 digest. Its root README is the Hub model
|
| 60 |
+
card; the working-tree README remains the GitHub landing page.
|
| 61 |
+
|
| 62 |
+
This export is a snapshot. If you edit any public source or documentation after
|
| 63 |
+
preparation, create a fresh versioned directory and use that path below:
|
| 64 |
+
|
| 65 |
+
```sh
|
| 66 |
+
python3 tools/prepare_public_release.py --target huggingface --output releases/huggingface-source-v9
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
The exporter refuses to overwrite an existing directory. Never point the upload
|
| 70 |
+
command at the development checkout or an old export containing stale terms.
|
| 71 |
+
|
| 72 |
+
## 3. Upload
|
| 73 |
+
|
| 74 |
+
From the Vons checkout, after reviewing the v8 export:
|
| 75 |
+
|
| 76 |
+
```sh
|
| 77 |
+
hf upload INLEVEL9/Vons releases/huggingface-source-v8 . \
|
| 78 |
+
--repo-type model \
|
| 79 |
+
--exclude RELEASE_MANIFEST.json \
|
| 80 |
+
--commit-message "Publish Vons source preview and reviewed research PDFs"
|
| 81 |
+
```
|
| 82 |
+
|
| 83 |
+
The final `.` places the files at the repository root while preserving nested
|
| 84 |
+
paths. The command creates a Hub commit directly; a separate Git commit/push
|
| 85 |
+
is unnecessary. It updates matching remote paths, including README and LICENSE,
|
| 86 |
+
and does not request deletion of unrelated remote files. The manifest stays
|
| 87 |
+
local as an audit receipt; it is not an additional source file in the allowlist.
|
| 88 |
+
|
| 89 |
+
A successful command prints a repository or commit URL. For a 401/403 response,
|
| 90 |
+
check the account, `INLEVEL9/Vons` write permission and credential scope; do not
|
| 91 |
+
create a duplicate personal repository. Official
|
| 92 |
+
[upload guidance](https://huggingface.co/docs/huggingface_hub/guides/upload)
|
| 93 |
+
describes folder uploads and filtering.
|
| 94 |
+
|
| 95 |
+
## 4. Verify the result
|
| 96 |
+
|
| 97 |
+
Open [the model card](https://huggingface.co/INLEVEL9/Vons) and
|
| 98 |
+
[Files and versions](https://huggingface.co/INLEVEL9/Vons/tree/main):
|
| 99 |
+
|
| 100 |
+
- The card says this is a source preview with no pretrained weights or hosted inference.
|
| 101 |
+
- License metadata uses `other`, `vons-community-commercial-1.0` and `LICENSE`.
|
| 102 |
+
It must no longer identify the software as Apache-2.0 or CC BY 4.0.
|
| 103 |
+
- Root `LICENSE` and `sdk/typescript/LICENSE` contain the same custom terms.
|
| 104 |
+
- Source paths such as `vons/`, `tools/`, `tests/` and `sdk/typescript/` are present.
|
| 105 |
+
- The Paper and Tech Report links open the 15-page paper and 20-page report.
|
| 106 |
+
Their digests match the publication index. No manuscript source bundle or
|
| 107 |
+
private review material is uploaded.
|
| 108 |
+
- Logo/documentation links resolve, and a signed-out visitor can read the card.
|
| 109 |
+
|
| 110 |
+
Save the resulting Hub commit URL with the local release manifest. GitHub
|
| 111 |
+
publication, manuscript publication and a hosted demo are separate actions.
|
| 112 |
+
For subsequent source changes, create a new reviewed export and repeat the same
|
| 113 |
+
upload command with its new path. Add research files only after their separate
|
| 114 |
+
v1.0 review and an explicit public-allowlist update.
|
| 115 |
+
|
| 116 |
+
The owner performs authentication, commit, push and publication. This local
|
| 117 |
+
preparation has not executed an upload or changed remote visibility.
|
| 118 |
+
Official CLI/upload guidance checked 2026-09-25.
|
docs/HUGGING_FACE.md
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: other
|
| 3 |
+
license_name: vons-community-commercial-1.0
|
| 4 |
+
license_link: LICENSE
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
tags:
|
| 8 |
+
- vons
|
| 9 |
+
- research
|
| 10 |
+
- candidate-selection
|
| 11 |
+
- onnx
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+

|
| 15 |
+
|
| 16 |
+
# Vons
|
| 17 |
+
|
| 18 |
+
**Plan globally. Decide locally. Keep the host in control.**
|
| 19 |
+
|
| 20 |
+
Compact local decision models for frontier-agent workflows.
|
| 21 |
+
|
| 22 |
+
**Kwangseob Ahn** · **INLEVEL9 / SEJONG UNIV.** · **oswarld@inlevel9.com**
|
| 23 |
+
|
| 24 |
+
## Source preview
|
| 25 |
+
|
| 26 |
+
This repository contains research source and documentation. It does not contain
|
| 27 |
+
pretrained model weights, tokenizer assets or datasets. Hosted inference and
|
| 28 |
+
`from_pretrained` loading are not available from this source preview.
|
| 29 |
+
|
| 30 |
+
Vons studies Direct and Diffusion candidate scoring with abstention. The host
|
| 31 |
+
owns execution, permission and consent. Model output never authorizes an action.
|
| 32 |
+
English-first operation and a model asset budget under 64 MiB are design targets;
|
| 33 |
+
they are not general quality or complete browser-package guarantees.
|
| 34 |
+
|
| 35 |
+
See [the model card](../docs/MODEL_CARD.md),
|
| 36 |
+
[the TypeScript SDK](../sdk/typescript/README.md),
|
| 37 |
+
[the paper and Tech Report](../docs/publications/README.md), and
|
| 38 |
+
[release preparation](../docs/PUBLIC_RELEASE.md).
|
| 39 |
+
|
| 40 |
+
## Read, try and contribute
|
| 41 |
+
|
| 42 |
+
This source preview is distributed through [INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons).
|
| 43 |
+
Source and contribution reports belong in
|
| 44 |
+
[inlevel9-com/Vons](https://github.com/inlevel9-com/Vons). The reviewed paper and Tech Report are included as PDF downloads below. The software remains a 0.1.0
|
| 45 |
+
research preview. No pretrained weights or working Space are announced.
|
| 46 |
+
|
| 47 |
+
Follow the [community guide](../docs/COMMUNITY.md) for the no-weight walkthrough,
|
| 48 |
+
experience and reproduction reports, and the
|
| 49 |
+
[contribution guide](../CONTRIBUTING.md) for review expectations. The paper is
|
| 50 |
+
being prepared for [arXiv](../docs/publications/ARXIV.md). User reports are not
|
| 51 |
+
independent research validation.
|
| 52 |
+
|
| 53 |
+
Maintainers can use the [Hub upload guide](../docs/HUB_PUBLISHING.md) to prepare
|
| 54 |
+
an organization card, source preview and later a separately validated Space.
|
| 55 |
+
|
| 56 |
+
## Paper and Tech Report
|
| 57 |
+
|
| 58 |
+
| Publication | Download |
|
| 59 |
+
| --- | --- |
|
| 60 |
+
| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows — paper v1.0, final editorial revision | [PDF · 15 pages](../docs/publications/vons-paper-v1.0.pdf) |
|
| 61 |
+
| Vons: Compact Decision Models for Frontier-Agent Workflows — Technical Report v1.0.1 | [PDF · 20 pages](../docs/publications/vons-tech-report-v1.0.1.pdf) |
|
| 62 |
+
|
| 63 |
+
Articles and original figures use **CC BY 4.0**. Their version, SHA-256 digests
|
| 64 |
+
and verification scope are in the [publication index](../docs/publications/README.md).
|
| 65 |
+
These preprints preserve negative results and missing measurements. No arXiv
|
| 66 |
+
identifier or peer-review acceptance is announced.
|
| 67 |
+
|
| 68 |
+
## Local use
|
| 69 |
+
|
| 70 |
+
```sh
|
| 71 |
+
python3 -m venv .venv
|
| 72 |
+
. .venv/bin/activate
|
| 73 |
+
python -m pip install -e '.[dev]'
|
| 74 |
+
python -m vons.cli generate-smoke --output data/generated/smoke.jsonl
|
| 75 |
+
python -m vons.cli validate-data --input data/generated/smoke.jsonl
|
| 76 |
+
```
|
| 77 |
+
|
| 78 |
+
The source export includes the research tools and pinned pilot configurations.
|
| 79 |
+
Obtain and review upstream assets separately before running model experiments.
|
| 80 |
+
Raw restricted benchmark data and private evidence are not redistributed here.
|
| 81 |
+
|
| 82 |
+
## Terms
|
| 83 |
+
|
| 84 |
+
Source is distributed under [Vons Community and Commercial License 1.0](../LICENSE):
|
| 85 |
+
qualifying noncommercial use is free; enterprise and other commercial use require
|
| 86 |
+
a separate written agreement. Contact oswarld@inlevel9.com for commercial terms.
|
| 87 |
+
This is source-available software, not OSI-approved open source.
|
| 88 |
+
|
| 89 |
+
The paper and Tech Report are distributed under [CC BY 4.0](../docs/publications/LICENSE.md).
|
| 90 |
+
This permits commercial article reuse with attribution; software rights are
|
| 91 |
+
separate. Third-party assets retain their own terms. See
|
| 92 |
+
[licensing scope](../docs/LICENSING.md). This source release does not establish research validation or a commercial
|
| 93 |
+
customer agreement.
|
docs/KASI_INTEGRATION.md
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# KASI integration in Vons
|
| 2 |
+
|
| 3 |
+
KASI is the first Vons domain adapter, not a separate model product. Vons owns
|
| 4 |
+
the general candidate judgment contract; the adapter preserves KASI's public
|
| 5 |
+
five-action contract for tool-calling hosts:
|
| 6 |
+
|
| 7 |
+
`call`, `clarify`, `confirm`, `refuse`, `respond`.
|
| 8 |
+
|
| 9 |
+
The model may propose ordinary calls, clarification, refusal, or a response. It
|
| 10 |
+
cannot grant consent. The host wrapper requires a host-owned registry of tool
|
| 11 |
+
names and risk levels, validates tools and arguments, applies confidence
|
| 12 |
+
thresholds, and derives `confirm` from that policy. The proposal's `risk` is
|
| 13 |
+
only advisory: it can never lower the registered host risk, and missing or
|
| 14 |
+
unknown risk is fail-closed. The host remains the only execution boundary.
|
| 15 |
+
|
| 16 |
+
Consent is bound to the complete call, including its arguments. Pass the
|
| 17 |
+
`KASICall.fingerprint()` returned for the exact call in `confirmed_calls`; a
|
| 18 |
+
tool name in `confirmed_tools` is retained only for compatibility and cannot
|
| 19 |
+
authorize a high-risk call.
|
| 20 |
+
|
| 21 |
+
The Python adapter stores a SHA-256 fingerprint. The TypeScript SDK exposes the
|
| 22 |
+
same canonical `{name, arguments}` consent key as a portable string because
|
| 23 |
+
browser runtimes do not provide synchronous SHA-256. A host that shares consent
|
| 24 |
+
records between the Python and TypeScript adapters must hash the TypeScript key
|
| 25 |
+
with SHA-256 before persistence and comparison.
|
| 26 |
+
|
| 27 |
+
Unregistered tools are refused. A minimal host setup is:
|
| 28 |
+
|
| 29 |
+
```python
|
| 30 |
+
from vons import KASIAdapter, KASICall, KASIProposal
|
| 31 |
+
|
| 32 |
+
adapter = KASIAdapter(tool_policy={"read_weather": "low", "unlock": "high"})
|
| 33 |
+
proposal = KASIProposal(
|
| 34 |
+
calls=(KASICall("read_weather", {"city": "Seoul"}),),
|
| 35 |
+
confidence=0.9,
|
| 36 |
+
risk="low",
|
| 37 |
+
)
|
| 38 |
+
decision = adapter.decide(proposal)
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
The mapping is intentionally one-way: KASI calls can be represented as Vons
|
| 42 |
+
choices, but arbitrary Vons choices do not become tool calls without a host
|
| 43 |
+
adapter. Existing KASI datasets, safety policy, and C99 adapter can be imported
|
| 44 |
+
behind a manifest when their provenance and licensing are recorded.
|
docs/LICENSING.md
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Licensing and distribution scope
|
| 2 |
+
|
| 3 |
+
The owner selected these terms on 2026-09-24. This local preparation is ready
|
| 4 |
+
for owner-controlled source distribution:
|
| 5 |
+
|
| 6 |
+
| Material | Terms |
|
| 7 |
+
|---|---|
|
| 8 |
+
| Vons-owned source, tests, configurations and software documentation | [Vons Community and Commercial License 1.0](../LICENSE) |
|
| 9 |
+
| Paper, Tech Report and their original research figures | [CC BY 4.0](publications/LICENSE.md) |
|
| 10 |
+
| Third-party libraries, upstream encoders/tokenizers and datasets | Their original terms; no relicense |
|
| 11 |
+
| INLEVEL9 logo and other marks | No general trademark license or endorsement |
|
| 12 |
+
| Model weights, restricted data and private evidence | Excluded from this source release |
|
| 13 |
+
|
| 14 |
+
The free software grant covers qualifying noncommercial personal, educational,
|
| 15 |
+
nonprofit research and public-benefit use. Enterprise use, including internal
|
| 16 |
+
evaluation, and other commercial use require a separate written agreement.
|
| 17 |
+
Paid consulting, client work, SaaS/API hosting and embedding in a commercial
|
| 18 |
+
product fall in that category. Contact [oswarld@inlevel9.com](mailto:oswarld@inlevel9.com)
|
| 19 |
+
for the scope and commercial terms; no price or automatic trial is specified.
|
| 20 |
+
|
| 21 |
+
This is **source-available software**, not OSI-approved open source. The article
|
| 22 |
+
license is deliberately separate: CC BY 4.0 permits commercial article reuse
|
| 23 |
+
with its attribution conditions, without granting commercial software rights.
|
| 24 |
+
Affiliation is attribution and does not establish institutional endorsement.
|
| 25 |
+
|
| 26 |
+
## Transition and distribution
|
| 27 |
+
|
| 28 |
+
Earlier local Python metadata declared Apache-2.0. This source release
|
| 29 |
+
uses `LicenseRef-Vons-Community-Commercial-1.0` and includes its exact text.
|
| 30 |
+
Frozen research snapshots retain their historical metadata and measurements.
|
| 31 |
+
This change makes no claim that rights previously granted under Apache-2.0 can
|
| 32 |
+
be withdrawn. Existing third-party licenses and notices remain effective.
|
| 33 |
+
|
| 34 |
+
The root license contains the custom terms for this source distribution.
|
| 35 |
+
A separate written agreement is required for enterprise or commercial software
|
| 36 |
+
use; publishing the source does not execute a customer contract. The source
|
| 37 |
+
list and hashes identify the reviewed files, and do not establish rights over
|
| 38 |
+
an unreviewed third-party asset. No manuscript or model weights accompany this
|
| 39 |
+
source release.
|
| 40 |
+
|
| 41 |
+
Commercial contracts, pricing, support and optional conformance services are
|
| 42 |
+
separate from this free community grant. No certification claim is made.
|
docs/MODEL_CARD.md
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Vons model card
|
| 2 |
+
|
| 3 |
+
Status: research source preview. This package contains no pretrained weights,
|
| 4 |
+
tokenizer files or benchmark datasets.
|
| 5 |
+
|
| 6 |
+
## Intended use
|
| 7 |
+
|
| 8 |
+
Vons studies bounded candidate selection and abstention in agent workflows.
|
| 9 |
+
A host supplies the state and candidates. The host alone enforces permission,
|
| 10 |
+
consent and execution policy. KASI is a separate compatibility adapter.
|
| 11 |
+
|
| 12 |
+
## Architecture and target
|
| 13 |
+
|
| 14 |
+
The research implementation offers a Direct scorer and a conditional Diffusion
|
| 15 |
+
scorer. Pilot configurations pin `google/bert_uncased_L-4_H-256_A-4` at revision
|
| 16 |
+
`387825ce42dbb39b87911cdf8e383ee3b25184f8`. This identifies a research dependency;
|
| 17 |
+
its assets are not included or relicensed here.
|
| 18 |
+
|
| 19 |
+
English-first input and a model asset budget below 64 MiB are design targets.
|
| 20 |
+
Runtime libraries, complete download size and memory use are distinct quantities.
|
| 21 |
+
|
| 22 |
+
## Evaluation and limitations
|
| 23 |
+
|
| 24 |
+
Historical local experiments include synthetic pilots and an evaluation-only
|
| 25 |
+
external benchmark diagnostic. They do not establish general task quality,
|
| 26 |
+
production readiness, calibrated safety or a Direct/Diffusion superiority claim.
|
| 27 |
+
Reviewed numerical claims belong to the versioned
|
| 28 |
+
[paper and Tech Report](publications/README.md), with exact evidence references.
|
| 29 |
+
Unreviewed historical tables are not distributed as current results.
|
| 30 |
+
|
| 31 |
+
The ONNX candidate head does not support general score questions. Python and
|
| 32 |
+
TypeScript token-budget checks are not fully identical. Browser smoke, repeated
|
| 33 |
+
latency, full numerical parity and memory evidence remain separate gates.
|
| 34 |
+
|
| 35 |
+
Do not use model confidence as a permission grant or assume it is calibrated.
|
| 36 |
+
Vons cannot execute tools or approve high-risk actions.
|
| 37 |
+
|
| 38 |
+
## Data and provenance
|
| 39 |
+
|
| 40 |
+
Synthetic data can be generated from the source. Restricted data, model-generated
|
| 41 |
+
raw responses, private prompts, model checkpoints and local manifests are
|
| 42 |
+
excluded from this distribution. Obtain any external dataset under its own
|
| 43 |
+
terms. The source release manifest records hashes of the files actually exported.
|
| 44 |
+
|
| 45 |
+
## Author and terms
|
| 46 |
+
|
| 47 |
+
Kwangseob Ahn — INLEVEL9 / SEJONG UNIV. — oswarld@inlevel9.com.
|
| 48 |
+
|
| 49 |
+
The Vons-owned source is prepared under [Vons Community and Commercial License
|
| 50 |
+
1.0](../LICENSE): noncommercial community use is free; enterprise/commercial use
|
| 51 |
+
requires a separate written agreement. The articles and their original figures
|
| 52 |
+
use [CC BY 4.0](publications/LICENSE.md). Third-party assets retain their own
|
| 53 |
+
terms, and no model weights are licensed or included by this source-only card.
|
| 54 |
+
See [licensing scope](LICENSING.md) and [release preparation](PUBLIC_RELEASE.md).
|
docs/PUBLIC_RELEASE.md
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Public source release
|
| 2 |
+
|
| 3 |
+
The public source set is defined by `configs/public-release.json`. The root
|
| 4 |
+
`.gitignore` excludes unreviewed root paths and internal documents; the release
|
| 5 |
+
tool independently rejects any Git candidate outside that exact file list.
|
| 6 |
+
|
| 7 |
+
## Included and excluded material
|
| 8 |
+
|
| 9 |
+
Included: Python and TypeScript source, tests, source demo files, the SDK
|
| 10 |
+
dependency lockfile, pilot configuration, reviewed public documentation and the
|
| 11 |
+
owner-supplied INLEVEL9 logo. Reviewed publications are listed separately in
|
| 12 |
+
`docs/publications/README.md`.
|
| 13 |
+
|
| 14 |
+
Excluded: all local datasets (including synthetic outputs), model weights,
|
| 15 |
+
tokenizers, ONNX external-data shards, experiment reports, evidence archives,
|
| 16 |
+
private prompts and raw model responses, agent/session records, license drafts,
|
| 17 |
+
historical planning documents, generated demo JavaScript, environments,
|
| 18 |
+
credentials, service caches and unreviewed binary files. Keep excluded originals
|
| 19 |
+
locally; never force-add them to make an upload work.
|
| 20 |
+
|
| 21 |
+
## Audit and export
|
| 22 |
+
|
| 23 |
+
Run from the repository root with Python 3.10+ and Git:
|
| 24 |
+
|
| 25 |
+
```sh
|
| 26 |
+
python tools/prepare_public_release.py
|
| 27 |
+
python tools/prepare_public_release.py --target github --output releases/github-source
|
| 28 |
+
python tools/prepare_public_release.py --target huggingface --output releases/huggingface-source
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
Each export is a new directory containing only allowlisted files and a
|
| 32 |
+
`RELEASE_MANIFEST.json` with byte counts and SHA-256 hashes. Existing output
|
| 33 |
+
directories are never overwritten. Use a new versioned output path after any
|
| 34 |
+
source change. The Hugging Face export substitutes its model card as the root
|
| 35 |
+
README. Both exports contain source and the two exact reviewed publication PDFs, with
|
| 36 |
+
no pretrained assets.
|
| 37 |
+
|
| 38 |
+
The audit detects known token/key patterns, personal home paths, private writing
|
| 39 |
+
session URLs, unexpected Git files, symlinks and changed reviewed binaries.
|
| 40 |
+
Pattern scanning is heuristic: passing it is not a complete secret audit or
|
| 41 |
+
proof of redistribution rights. Read the actual export before publishing.
|
| 42 |
+
|
| 43 |
+
## GitHub
|
| 44 |
+
|
| 45 |
+
The source checkout uses `main`; review `git status`, the existing commit history
|
| 46 |
+
and the export manifest before publishing changes. The owner performs commits,
|
| 47 |
+
pushes and visibility changes. This preparation does not perform those actions. Preserve a clean initial tree: `.gitignore` cannot
|
| 48 |
+
remove already committed material from history. If existing remote history is
|
| 49 |
+
later attached, review that history independently before making it public.
|
| 50 |
+
|
| 51 |
+
The CI workflow checks the release allowlist, Python tests and lint, and the SDK
|
| 52 |
+
tests and three browser builds. Push/pull-request checks run when source,
|
| 53 |
+
tests, configuration or the workflow changes; documentation-only edits do not
|
| 54 |
+
trigger the full suite. A manual run remains available. SDK tokenizer tests use
|
| 55 |
+
synthetic fixtures and require no private pilot assets. It has read-only repository permissions and
|
| 56 |
+
does not publish packages or artifacts. Checks that require unavailable optional
|
| 57 |
+
Python dependencies may be skipped; install training/export extras for the full
|
| 58 |
+
local suite. The historical pilot-summary regression is also skipped when its
|
| 59 |
+
private input files are absent. A separate synthetic regression runs without
|
| 60 |
+
those files; do not publish the historical inputs merely to remove that skip.
|
| 61 |
+
|
| 62 |
+
## Hugging Face
|
| 63 |
+
|
| 64 |
+
Upload only a reviewed versioned Hugging Face export to `INLEVEL9/Vons`.
|
| 65 |
+
The current preparation is `releases/huggingface-source-v8`; see the
|
| 66 |
+
[step-by-step upload guide](HUB_PUBLISHING.md). Do not select the development workspace in the
|
| 67 |
+
Hub upload dialog. Use explicit allow/ignore patterns if using `upload_folder`;
|
| 68 |
+
its handling of ignore files depends on the client version and upload method.
|
| 69 |
+
|
| 70 |
+
This source preview has no weights and no inference widget. A later weight
|
| 71 |
+
release needs its own exact file inventory, upstream notices, model/data
|
| 72 |
+
provenance and runtime validation. Do not turn the ignored checkpoint directory
|
| 73 |
+
into a public upload by adding Git LFS patterns.
|
| 74 |
+
|
| 75 |
+
Official references: [GitHub ignore guidance](https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files)
|
| 76 |
+
and [Hub upload guidance](https://huggingface.co/docs/huggingface_hub/guides/upload).
|
| 77 |
+
|
| 78 |
+
## Terms and publication state
|
| 79 |
+
|
| 80 |
+
The owner selected noncommercial-free / enterprise-commercial software terms
|
| 81 |
+
and CC BY 4.0 articles on 2026-09-24. The prepared root [license](../LICENSE), SDK
|
| 82 |
+
license copy, package metadata and Hub card now match that source-available
|
| 83 |
+
direction. [Licensing scope](LICENSING.md) explains the boundary and preserves
|
| 84 |
+
earlier and third-party rights. Local preparation is complete for the owner to
|
| 85 |
+
commit, push and publish; no commercial customer agreement is adopted here.
|
| 86 |
+
npm publishing remains disabled with `private: true`.
|
| 87 |
+
|
| 88 |
+
The Python build backend requires setuptools 77.0.3+ to include the custom SPDX
|
| 89 |
+
license reference and exact license file in wheel/sdist metadata. No runtime
|
| 90 |
+
dependency was added. Verify built package contents as well as source exports.
|
| 91 |
+
|
| 92 |
+
The author-approved public contact is Kwangseob Ahn, oswarld@inlevel9.com,
|
| 93 |
+
INLEVEL9 / SEJONG UNIV. Logo inclusion does not grant trademark rights or imply
|
| 94 |
+
institutional endorsement. Model-assisted writing/review is not peer acceptance.
|
docs/assets/inlevel9-signature-dark.png
ADDED
|
docs/assets/inlevel9-signature.png
ADDED
|
docs/publications/ARXIV.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# arXiv preparation
|
| 2 |
+
|
| 3 |
+
The [reviewed paper PDF](vons-paper-v1.0.pdf) is included in the source release,
|
| 4 |
+
alongside the [Technical Report](vons-tech-report-v1.0.1.pdf). arXiv submission
|
| 5 |
+
is being prepared; no arXiv identifier or acceptance is announced. arXiv performs
|
| 6 |
+
content moderation, not journal peer review; see its
|
| 7 |
+
[moderation policy](https://info.arxiv.org/help/moderation/index.html).
|
| 8 |
+
|
| 9 |
+
## Current preparation
|
| 10 |
+
|
| 11 |
+
Exact local PDF hashes match the reviewed final manuscripts, and all 35 pages
|
| 12 |
+
have been inspected. The documents were generated with Typst, not TeX/LaTeX.
|
| 13 |
+
The paper is being prepared for the genuine non-TeX PDF submission route,
|
| 14 |
+
subject to arXiv's processing checks. An independent source rebuild remains
|
| 15 |
+
unverified; see the [publication index](README.md) for exact verification scope.
|
| 16 |
+
Historical drafts and claim-to-source records remain private.
|
| 17 |
+
|
| 18 |
+
## Research scope
|
| 19 |
+
|
| 20 |
+
Present the bounded decision contract, engineering evidence and informative
|
| 21 |
+
negative results. State the synthetic pilot's label imbalance and baseline,
|
| 22 |
+
external-evaluation failures, token budget and browser-verification limitations
|
| 23 |
+
alongside the relevant results. A narrower research preprint can describe open
|
| 24 |
+
engineering gates without claiming that those gates have passed or that a model
|
| 25 |
+
is ready for deployment. Preserve a separate private claim-to-source audit.
|
| 26 |
+
|
| 27 |
+
The publication language is English. Author metadata is **Kwangseob Ahn**,
|
| 28 |
+
**INLEVEL9 / SEJONG UNIV.**, **oswarld@inlevel9.com**. Article preparation uses
|
| 29 |
+
[CC BY 4.0](LICENSE.md); software terms are separate. The author remains
|
| 30 |
+
responsible for scientific claims, references and model-assistance disclosure.
|
| 31 |
+
|
| 32 |
+
## Files and checks
|
| 33 |
+
|
| 34 |
+
Prepare a readable PDF, editable manuscript source, included figures and
|
| 35 |
+
bibliography, together with a reproducible build instruction. Verify every
|
| 36 |
+
source-bundle entry and record exact hashes. Resolve missing references,
|
| 37 |
+
orphan captions, table breaks and unavailable figures; inspect all PDF pages.
|
| 38 |
+
Keep private claim ledgers, prompts, review conversations, restricted benchmark
|
| 39 |
+
rows and unrelated repository files outside the submission bundle.
|
| 40 |
+
|
| 41 |
+
arXiv prefers TeX/LaTeX and requires source when the PDF was made from TeX/LaTeX.
|
| 42 |
+
Include figure files instead of external figure links. See the official
|
| 43 |
+
[submission overview](https://info.arxiv.org/help/submit/index.html).
|
| 44 |
+
For a genuine non-TeX PDF, follow the [PDF requirements](https://info.arxiv.org/help/submit_pdf.html),
|
| 45 |
+
including embedded outline fonts and no encryption or active content.
|
| 46 |
+
For a TeX package, include the necessary bibliography source or compatible
|
| 47 |
+
generated bibliography and confirm the top-level file builds from the bundle
|
| 48 |
+
root. Inspect arXiv's processed PDF as well as the local build. See
|
| 49 |
+
[TeX submission guidance](https://info.arxiv.org/help/submit_tex.html).
|
| 50 |
+
|
| 51 |
+
## Author submission handoff
|
| 52 |
+
|
| 53 |
+
1. Review the exact final manuscript, source bundle, hashes and remaining limits.
|
| 54 |
+
2. Confirm title, abstract, author affiliation, article rights and a subject
|
| 55 |
+
category based on the paper's actual contribution. `cs.AI` and `cs.LG` are
|
| 56 |
+
candidates to assess, not confirmed classifications.
|
| 57 |
+
3. Check the account and any applicable [endorsement requirement](https://info.arxiv.org/help/endorsement.html).
|
| 58 |
+
4. Prepare the upload and metadata; verify the processed PDF and the selected
|
| 59 |
+
[arXiv license](https://info.arxiv.org/help/license/index.html). CC BY 4.0 is
|
| 60 |
+
an available option consistent with the selected article terms.
|
| 61 |
+
5. The author approves the concrete submission and completes the submission
|
| 62 |
+
agreement. Add the real arXiv URL to the publication index only after it exists.
|
| 63 |
+
|
| 64 |
+
Official guidance was checked on 2026-09-25; recheck it at submission time.
|
| 65 |
+
The GitHub destination is [inlevel9-com/Vons](https://github.com/inlevel9-com/Vons)
|
| 66 |
+
and the Hugging Face repository is [INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons).
|
| 67 |
+
Neither link establishes public artifact availability, replication or acceptance.
|
docs/publications/LICENSE.md
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Article license: CC BY 4.0
|
| 2 |
+
|
| 3 |
+
The Vons paper and Tech Report, including their original research figures, are
|
| 4 |
+
prepared for release under the **Creative Commons Attribution 4.0 International
|
| 5 |
+
License (CC BY 4.0)**, as selected by the author on 2026-09-24.
|
| 6 |
+
|
| 7 |
+
Author: Kwangseob Ahn
|
| 8 |
+
Affiliation: INLEVEL9 / SEJONG UNIV.
|
| 9 |
+
Contact: oswarld@inlevel9.com
|
| 10 |
+
|
| 11 |
+
The [CC BY 4.0 legal code](https://creativecommons.org/licenses/by/4.0/legalcode.en)
|
| 12 |
+
governs the article material identified by this notice. The
|
| 13 |
+
[license summary](https://creativecommons.org/licenses/by/4.0/) explains sharing,
|
| 14 |
+
adaptation and attribution. Commercial reuse of the articles is permitted under
|
| 15 |
+
CC BY 4.0; the software's commercial-use restriction does not apply to them.
|
| 16 |
+
|
| 17 |
+
Retain the author, title, version, license link and supplied attribution notices;
|
| 18 |
+
indicate modifications. Do not imply the author's or institutions' endorsement.
|
| 19 |
+
Third-party material specifically identified under other terms, software source
|
| 20 |
+
excerpts identified as separately licensed, logos, raw benchmark data and private
|
| 21 |
+
verification records are excluded from this article grant. Statutory exceptions
|
| 22 |
+
and independent rights remain unaffected.
|
| 23 |
+
|
| 24 |
+
This notice does not license the Vons software or any model weights. See the
|
| 25 |
+
repository's [software license](../../LICENSE) for software use. It does not
|
| 26 |
+
claim that an article has been submitted, accepted, externally peer-reviewed or
|
| 27 |
+
already uploaded to a public destination.
|
| 28 |
+
|
| 29 |
+
## Notice for the v0.2 release copies
|
| 30 |
+
|
| 31 |
+
This notice records the author's decision after the research evidence was
|
| 32 |
+
frozen. It applies to the selected release copies of the Vons paper v0.2 and
|
| 33 |
+
Technical Report v0.2 and their original figures. Where an earlier manuscript
|
| 34 |
+
states that article terms await an owner decision, this notice supersedes that
|
| 35 |
+
article-license status only. It does not alter measurements, claim provenance,
|
| 36 |
+
publication status, or the remaining research gates.
|
| 37 |
+
|
| 38 |
+
Distribute this notice alongside the reviewed manuscript files and their
|
| 39 |
+
release checksums. The Tech Report's original author/model attribution is
|
| 40 |
+
preserved; a separate licensing notice does not replace its authoring history.
|
docs/publications/README.md
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Paper and Tech Report
|
| 2 |
+
|
| 3 |
+
The reviewed PDFs are included in this release. Both files preserve the final
|
| 4 |
+
manuscript bytes; publication does not imply new experiments or peer acceptance.
|
| 5 |
+
|
| 6 |
+
Author: **Kwangseob Ahn**
|
| 7 |
+
Affiliation: **INLEVEL9 / SEJONG UNIV.**
|
| 8 |
+
Contact: **oswarld@inlevel9.com**
|
| 9 |
+
|
| 10 |
+
| Publication | Version | Download |
|
| 11 |
+
| --- | --- | --- |
|
| 12 |
+
| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows | Paper v1.0, final editorial revision | [PDF · 15 pages](vons-paper-v1.0.pdf) |
|
| 13 |
+
| Vons: Compact Decision Models for Frontier-Agent Workflows | Technical Report v1.0.1 | [PDF · 20 pages](vons-tech-report-v1.0.1.pdf) |
|
| 14 |
+
|
| 15 |
+
## Verification and provenance
|
| 16 |
+
|
| 17 |
+
The final documents were reviewed against the frozen evidence packet and primary
|
| 18 |
+
references, including tables, figures, negative results and missing measurements.
|
| 19 |
+
Every PDF page was visually inspected. Independently checked local SHA-256
|
| 20 |
+
values match the reviewed final versions. All pages render as US Letter, fonts
|
| 21 |
+
are embedded, and no PDF JavaScript, launch action or embedded attachment was
|
| 22 |
+
found. These are genuine Typst-generated PDFs; a clean source rebuild has not
|
| 23 |
+
been independently completed and is not claimed.
|
| 24 |
+
|
| 25 |
+
| File | SHA-256 |
|
| 26 |
+
| --- | --- |
|
| 27 |
+
| `vons-paper-v1.0.pdf` | `ae0e09c60e17a98b52bb25c611b2489d10b18bd98d87974f2e8dd25d746a5fc8` |
|
| 28 |
+
| `vons-tech-report-v1.0.1.pdf` | `33076d9e52d4a073060066215dd390e3b0bf0cdef48c5ffb2a7c5189854254ed` |
|
| 29 |
+
|
| 30 |
+
Only these exact PDFs are permitted by the public allowlist. Drafts, prompts,
|
| 31 |
+
claim ledgers, review conversations, source bundles and restricted benchmark data
|
| 32 |
+
remain excluded. The software has its own version, **0.1.0 research preview**.
|
| 33 |
+
|
| 34 |
+
The [CC BY 4.0 article license](LICENSE.md) covers both manuscripts and their
|
| 35 |
+
original figures. Software rights are governed separately by the
|
| 36 |
+
[Vons Community and Commercial License 1.0](../../LICENSE); article licensing
|
| 37 |
+
does not grant commercial software rights.
|
| 38 |
+
|
| 39 |
+
arXiv submission is in preparation. No arXiv identifier, DOI or acceptance is
|
| 40 |
+
announced. See the [arXiv preparation checklist](ARXIV.md).
|
docs/publications/vons-paper-v1.0.pdf
ADDED
|
Binary file (131 Bytes). View file
|
|
|
docs/publications/vons-tech-report-v1.0.1.pdf
ADDED
|
Binary file (131 Bytes). View file
|
|
|
pyproject.toml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=77.0.3"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "vons"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
description = "Small local decision models for frontier-agent workflows"
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
requires-python = ">=3.10"
|
| 11 |
+
license = "LicenseRef-Vons-Community-Commercial-1.0"
|
| 12 |
+
license-files = ["LICENSE"]
|
| 13 |
+
authors = [{ name = "Vons contributors" }]
|
| 14 |
+
dependencies = []
|
| 15 |
+
|
| 16 |
+
[project.optional-dependencies]
|
| 17 |
+
train = ["numpy>=1.26", "torch>=2.2", "transformers>=4.40", "safetensors>=0.4"]
|
| 18 |
+
export = ["onnx>=1.16", "onnxruntime>=1.18", "onnxscript>=0.5"]
|
| 19 |
+
ollama = ["ollama>=0.3"]
|
| 20 |
+
dev = ["pytest>=8", "ruff>=0.6"]
|
| 21 |
+
|
| 22 |
+
[project.scripts]
|
| 23 |
+
vons = "vons.cli:main"
|
| 24 |
+
|
| 25 |
+
[tool.setuptools.packages.find]
|
| 26 |
+
include = ["vons*"]
|
| 27 |
+
|
| 28 |
+
[tool.ruff]
|
| 29 |
+
line-length = 100
|
| 30 |
+
target-version = "py310"
|
| 31 |
+
# Preserve acquired upstream evidence byte-for-byte; it is not Vons source.
|
| 32 |
+
extend-exclude = ["data/restricted"]
|
| 33 |
+
|
| 34 |
+
[tool.pytest.ini_options]
|
| 35 |
+
testpaths = ["tests"]
|
sdk/typescript/LICENSE
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Vons Community and Commercial License 1.0
|
| 2 |
+
SPDX-License-Identifier: LicenseRef-Vons-Community-Commercial-1.0
|
| 3 |
+
|
| 4 |
+
1. Scope and definitions
|
| 5 |
+
|
| 6 |
+
"Licensor" means each copyright holder who makes material available under this
|
| 7 |
+
license. "Software" means the Vons-owned source code, configuration, tests and
|
| 8 |
+
supporting software documentation accompanied by this license, including copies
|
| 9 |
+
and modifications. "You" means the individual or legal entity exercising the
|
| 10 |
+
permissions below, including persons acting on that entity's behalf.
|
| 11 |
+
|
| 12 |
+
This license does not cover material with a separate license notice. In
|
| 13 |
+
particular, the paper and Tech Report and their original research figures are
|
| 14 |
+
licensed separately under CC BY 4.0. Third-party dependencies, upstream model
|
| 15 |
+
assets, datasets, trademarks and logos are not relicensed by this document.
|
| 16 |
+
No model weights or benchmark datasets are supplied with this source release.
|
| 17 |
+
|
| 18 |
+
"Noncommercial Use" means personal learning, independent noncommercial
|
| 19 |
+
research, or noncommercial education, research or public-benefit activity by a
|
| 20 |
+
nonprofit or educational institution, where the activity is not undertaken for
|
| 21 |
+
commercial advantage, monetary compensation, or on behalf of a business.
|
| 22 |
+
Nonprofit or educational status alone does not make commercial activity
|
| 23 |
+
Noncommercial Use. Unrestricted donations that do not buy a service or benefit
|
| 24 |
+
do not by themselves make an otherwise qualifying activity commercial.
|
| 25 |
+
|
| 26 |
+
"Commercial Use" includes paid consulting or client work; use in or to support
|
| 27 |
+
a monetized product, service, SaaS, API, hosted or managed service; internal
|
| 28 |
+
business operations; and evaluation, research or deployment by, for, or on
|
| 29 |
+
behalf of a for-profit enterprise. Contractors and affiliates acting for such
|
| 30 |
+
an enterprise require the same permission. Personal use by an employee outside
|
| 31 |
+
their employment, without business benefit, may qualify as Noncommercial Use.
|
| 32 |
+
|
| 33 |
+
2. Free community permission
|
| 34 |
+
|
| 35 |
+
Subject to this license, each Licensor grants You a worldwide, nonexclusive,
|
| 36 |
+
royalty-free copyright license to use, reproduce, study, modify, display and
|
| 37 |
+
distribute the Software solely for Noncommercial Use. You may provide access
|
| 38 |
+
to an otherwise qualifying noncommercial service under these conditions.
|
| 39 |
+
Only rights that a Licensor is entitled to grant are included.
|
| 40 |
+
|
| 41 |
+
3. Redistribution and attribution
|
| 42 |
+
|
| 43 |
+
When distributing copies or substantial portions of the Software, including
|
| 44 |
+
modified copies or a compiled form, You must:
|
| 45 |
+
|
| 46 |
+
(a) retain this license, copyright notices and attribution notices;
|
| 47 |
+
(b) clearly identify Your modifications and preserve their provenance;
|
| 48 |
+
(c) identify the Vons version or source revision used where available; and
|
| 49 |
+
(d) convey these same use restrictions for the covered Software, without
|
| 50 |
+
representing that You can grant Commercial Use rights.
|
| 51 |
+
|
| 52 |
+
Do not remove notices, misrepresent origin, or imply endorsement by a Licensor,
|
| 53 |
+
INLEVEL9, Sejong University or any other named institution. For research that
|
| 54 |
+
uses Vons, include an appropriate citation to the software and relevant paper
|
| 55 |
+
or Tech Report when reporting results. An unavailable DOI is not required.
|
| 56 |
+
Attribution or citation does not authorize an otherwise restricted use.
|
| 57 |
+
|
| 58 |
+
4. Enterprise and commercial permission
|
| 59 |
+
|
| 60 |
+
Commercial Use is not granted by this license. It requires a separate written
|
| 61 |
+
commercial agreement from the relevant Licensor before the activity begins.
|
| 62 |
+
There is no automatic enterprise evaluation or free-trial exception. Contact
|
| 63 |
+
oswarld@inlevel9.com to arrange the scope, fees and permitted entities.
|
| 64 |
+
This license sets no price, billing unit, service level or support commitment.
|
| 65 |
+
A commercial agreement applies only to the parties and scope it identifies.
|
| 66 |
+
|
| 67 |
+
5. Outputs and independent rights
|
| 68 |
+
|
| 69 |
+
This license claims no ownership or royalty over Your inference outputs merely
|
| 70 |
+
because they were produced using the Software. Outputs do not authorize an
|
| 71 |
+
unlicensed deployment and may contain material subject to independent rights.
|
| 72 |
+
Your inputs remain subject to their own rights and permissions.
|
| 73 |
+
|
| 74 |
+
No trademark, logo, certification or endorsement rights are granted, except
|
| 75 |
+
the descriptive use needed to comply with the notices above. No patent license
|
| 76 |
+
is granted by this document. Statutory exceptions and other independently
|
| 77 |
+
obtained permissions are unaffected. This license does not revoke rights
|
| 78 |
+
previously granted under another license for an earlier copy, and does not
|
| 79 |
+
restrict separately licensed third-party components.
|
| 80 |
+
|
| 81 |
+
6. Termination and cure
|
| 82 |
+
|
| 83 |
+
Your permissions terminate automatically if You materially breach this
|
| 84 |
+
license. They are reinstated when the breach is cured within 30 days after
|
| 85 |
+
You first discover it, provided this is Your first breach. Otherwise written
|
| 86 |
+
reinstatement from the relevant Licensor is required. Termination does not
|
| 87 |
+
affect compliant recipients' direct permissions or obligations that by their
|
| 88 |
+
nature survive, including the notices and disclaimers below.
|
| 89 |
+
|
| 90 |
+
7. Warranty disclaimer and liability
|
| 91 |
+
|
| 92 |
+
TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE SOFTWARE IS PROVIDED
|
| 93 |
+
"AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WARRANTIES
|
| 94 |
+
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND
|
| 95 |
+
NONINFRINGEMENT. THE LICENSORS ARE NOT LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 96 |
+
LIABILITY ARISING FROM THE SOFTWARE OR ITS USE, WHETHER IN CONTRACT, TORT OR
|
| 97 |
+
OTHERWISE. NOTHING HERE EXCLUDES LIABILITY THAT CANNOT LAWFULLY BE EXCLUDED.
|
| 98 |
+
|
| 99 |
+
8. General
|
| 100 |
+
|
| 101 |
+
Rights not expressly granted are reserved. No later license version applies
|
| 102 |
+
automatically. If a provision is unenforceable, the remaining provisions remain
|
| 103 |
+
in effect to the extent permitted by law. This is a source-available license
|
| 104 |
+
with use restrictions; it is not an OSI-approved open-source license.
|
sdk/typescript/README.md
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# `@vons/decision-sdk`
|
| 2 |
+
|
| 3 |
+
The TypeScript package mirrors the Vons Python request/response contract and
|
| 4 |
+
keeps KASI policy and execution in the host. It contains no tool executor and
|
| 5 |
+
does not treat a model-produced risk or confidence value as permission.
|
| 6 |
+
|
| 7 |
+
The contract-only entry point remains small, while the optional `./onnx`
|
| 8 |
+
subpath uses pinned `onnxruntime-web@1.30.0` and
|
| 9 |
+
`@huggingface/tokenizers@0.2.0` for local browser inference. Applications
|
| 10 |
+
provide a model backend through `DecisionBackend`; the runtime helper verifies
|
| 11 |
+
bundle bytes, uses the declared WASM or WebGPU provider, and does not silently
|
| 12 |
+
fall back from an unavailable accelerator.
|
| 13 |
+
`callFingerprint` returns a canonical JSON consent key. Hosts that persist
|
| 14 |
+
consent for the Python adapter should SHA-256 this key before storage so the
|
| 15 |
+
cross-language fingerprint format remains identical.
|
| 16 |
+
|
| 17 |
+
Node 22+ can load the source directly with its TypeScript type stripping during
|
| 18 |
+
early development. A release build should type-check and bundle the package
|
| 19 |
+
with the consuming application's toolchain.
|
| 20 |
+
|
| 21 |
+
The local smoke harness covers both pilot bundles and both providers. Build it
|
| 22 |
+
and serve the repository root so the browser can fetch the ignored ORT runtime
|
| 23 |
+
files and the existing pilot artifacts:
|
| 24 |
+
|
| 25 |
+
```sh
|
| 26 |
+
npm ci
|
| 27 |
+
npm run build:browser
|
| 28 |
+
cd ../..
|
| 29 |
+
python3 -m http.server 8765 --bind 127.0.0.1
|
| 30 |
+
```
|
| 31 |
+
|
| 32 |
+
Open `http://127.0.0.1:8765/sdk/typescript/demo/`. The smoke page reports
|
| 33 |
+
session-load and one-request timings, optional Chromium JS-heap observations,
|
| 34 |
+
provider evidence, and the contract response. These are diagnostic smoke
|
| 35 |
+
values, not the repeated benchmark protocol.
|
| 36 |
+
|
| 37 |
+
## Source preview
|
| 38 |
+
|
| 39 |
+
This package is private in npm metadata until an npm release is requested.
|
| 40 |
+
Its `files` allowlist includes source, this guide and the software license. Models, tokenizers,
|
| 41 |
+
datasets, generated demo JavaScript and dependencies are intentionally excluded
|
| 42 |
+
from the public source export. Build the demos locally and supply your own
|
| 43 |
+
reviewed model bundles before opening the smoke, benchmark or parity pages.
|
| 44 |
+
|
| 45 |
+
The [Vons Community and Commercial License 1.0](LICENSE) permits qualifying
|
| 46 |
+
noncommercial use without a fee. Enterprise and other commercial use require a
|
| 47 |
+
separate written agreement; contact oswarld@inlevel9.com. This is source-available
|
| 48 |
+
software. Dependencies retain their own terms, and the research articles' CC BY
|
| 49 |
+
4.0 license does not grant commercial SDK rights.
|
sdk/typescript/demo/benchmark.html
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8" />
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
| 6 |
+
<title>Vons ONNX Runtime Web benchmark</title>
|
| 7 |
+
</head>
|
| 8 |
+
<body>
|
| 9 |
+
<main>
|
| 10 |
+
<h1>Vons ONNX Runtime Web benchmark</h1>
|
| 11 |
+
<p>Run with query parameters such as <code>?backend=direct&provider=wasm&sessions=5&warmup=10&repeats=100</code>.</p>
|
| 12 |
+
<button id="download-report" type="button" hidden>Download raw report</button>
|
| 13 |
+
<pre id="output">Waiting for benchmark parameters.</pre>
|
| 14 |
+
</main>
|
| 15 |
+
<script type="module" src="./benchmark.js"></script>
|
| 16 |
+
</body>
|
| 17 |
+
</html>
|
sdk/typescript/demo/benchmark.ts
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { createOnnxWebBackend, type OnnxTimingSample } from "../src/onnx.ts";
|
| 2 |
+
|
| 3 |
+
type BackendName = "direct" | "diffusion";
|
| 4 |
+
type ProviderName = "wasm" | "webgpu";
|
| 5 |
+
|
| 6 |
+
interface PerformanceWithMemory extends Performance {
|
| 7 |
+
memory?: { usedJSHeapSize: number };
|
| 8 |
+
}
|
| 9 |
+
|
| 10 |
+
interface NavigatorWithDeviceMemory extends Navigator {
|
| 11 |
+
deviceMemory?: number;
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
interface BenchmarkSample extends Omit<OnnxTimingSample, "live_candidates" | "allocated_candidates" | "live_tokens" | "sequence_length" | "tokenizeMs" | "prepareFeedMs" | "inferenceAndReadbackMs" | "postprocessMs" | "totalRequestMs"> {
|
| 15 |
+
run_id: string;
|
| 16 |
+
session_start: number;
|
| 17 |
+
repeat: number;
|
| 18 |
+
case_id: string;
|
| 19 |
+
backend: BackendName;
|
| 20 |
+
requested_provider: ProviderName;
|
| 21 |
+
observed_provider: ProviderName | null;
|
| 22 |
+
observed_provider_status: string;
|
| 23 |
+
live_candidates: number | null;
|
| 24 |
+
allocated_candidates: number | null;
|
| 25 |
+
live_tokens: number | null;
|
| 26 |
+
sequence_length: number | null;
|
| 27 |
+
tokenizeMs: number | null;
|
| 28 |
+
prepareFeedMs: number | null;
|
| 29 |
+
inferenceAndReadbackMs: number | null;
|
| 30 |
+
postprocessMs: number | null;
|
| 31 |
+
totalRequestMs: number | null;
|
| 32 |
+
bundle_hash: string | null;
|
| 33 |
+
tokenizer_hash: string | null;
|
| 34 |
+
input_hash: string;
|
| 35 |
+
noise_hash: string | null;
|
| 36 |
+
js_heap_used_bytes_before: number | null;
|
| 37 |
+
js_heap_used_bytes_after: number | null;
|
| 38 |
+
status: "ok" | "error";
|
| 39 |
+
error_kind?: string;
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
interface SessionRecord {
|
| 43 |
+
session_start: number;
|
| 44 |
+
load_ms: number;
|
| 45 |
+
status: "ok" | "error";
|
| 46 |
+
error_kind?: string;
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
const output = document.querySelector<HTMLElement>("#output");
|
| 50 |
+
if (!output) throw new Error("benchmark output is missing");
|
| 51 |
+
const downloadButton = document.querySelector<HTMLButtonElement>("#download-report");
|
| 52 |
+
if (!downloadButton) throw new Error("benchmark download button is missing");
|
| 53 |
+
|
| 54 |
+
function showReport(report: unknown, fileName: string): void {
|
| 55 |
+
const text = JSON.stringify(report, null, 2);
|
| 56 |
+
output.textContent = text;
|
| 57 |
+
downloadButton.hidden = false;
|
| 58 |
+
downloadButton.onclick = () => {
|
| 59 |
+
const blob = new Blob([text], { type: "application/json" });
|
| 60 |
+
const url = URL.createObjectURL(blob);
|
| 61 |
+
const anchor = document.createElement("a");
|
| 62 |
+
anchor.href = url;
|
| 63 |
+
anchor.download = fileName;
|
| 64 |
+
anchor.click();
|
| 65 |
+
URL.revokeObjectURL(url);
|
| 66 |
+
};
|
| 67 |
+
console.log(text);
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
const manifestUrls = {
|
| 71 |
+
direct: new URL("../../../artifacts/pilot/bundle-manifest-v1.json", import.meta.url),
|
| 72 |
+
diffusion: new URL("../../../artifacts/pilot-diffusion/bundle-manifest-v1.json", import.meta.url),
|
| 73 |
+
};
|
| 74 |
+
|
| 75 |
+
const manifestHashes = {
|
| 76 |
+
direct: "bd39eea2b8d5ed5f4d57daefdd4822c4e2c4817a68c4a42bef850b5edcbc97d",
|
| 77 |
+
diffusion: "df0e7cae640c6604fb517a88d08dbdaa2454d6b3dbbf5708926349cf490b9607",
|
| 78 |
+
};
|
| 79 |
+
|
| 80 |
+
const heapBytes = (): number | null => (performance as PerformanceWithMemory).memory?.usedJSHeapSize ?? null;
|
| 81 |
+
|
| 82 |
+
function parameter(name: string, fallback: number): number {
|
| 83 |
+
const raw = new URLSearchParams(location.search).get(name);
|
| 84 |
+
if (raw === null) return fallback;
|
| 85 |
+
const value = Number(raw);
|
| 86 |
+
if (!Number.isInteger(value) || value < 1) throw new RangeError(`${name} must be a positive integer`);
|
| 87 |
+
return value;
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
function textParameter(name: string, fallback: string): string {
|
| 91 |
+
return new URLSearchParams(location.search).get(name) ?? fallback;
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
function stableJson(value: unknown): string {
|
| 95 |
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
| 96 |
+
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
| 97 |
+
const entries = Object.entries(value as Record<string, unknown>).sort(([a], [b]) => a.localeCompare(b));
|
| 98 |
+
return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(",")}}`;
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
async function sha256(value: string | ArrayBufferLike): Promise<string> {
|
| 102 |
+
const bytes = typeof value === "string" ? new TextEncoder().encode(value) : new Uint8Array(value);
|
| 103 |
+
const digest = await crypto.subtle.digest("SHA-256", bytes as unknown as BufferSource);
|
| 104 |
+
return Array.from(new Uint8Array(digest), (item) => item.toString(16).padStart(2, "0")).join("");
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
// Frozen from NumPy default_rng(7).standard_normal((1, 32), dtype=float32).
|
| 108 |
+
// The array, rather than a language-specific seed algorithm, is the parity input.
|
| 109 |
+
const FROZEN_NOISE_SEED_7 = Float32Array.from([
|
| 110 |
+
1.52196932, -1.14410579, 1.15016162, -1.78465319, -0.77149874, 1.01916146,
|
| 111 |
+
-0.926134586, -0.415745318, -0.0629121512, 0.513251185, -0.285995334,
|
| 112 |
+
1.98480856, 1.30306911, -0.00464721443, 1.17548203, 1.34566987,
|
| 113 |
+
-0.171809882, -2.4133966, -0.0993812904, -0.979565322, 0.942675471,
|
| 114 |
+
0.452982962, 0.53610903, 0.494613141, 1.94684041, 0.207542002,
|
| 115 |
+
-1.64390635, -0.542155981, -0.384397745, -0.410856962, 0.946411192,
|
| 116 |
+
0.979099393,
|
| 117 |
+
]);
|
| 118 |
+
|
| 119 |
+
function percentile(values: number[], fraction: number): number | null {
|
| 120 |
+
if (values.length === 0) return null;
|
| 121 |
+
const sorted = [...values].sort((a, b) => a - b);
|
| 122 |
+
const index = (sorted.length - 1) * fraction;
|
| 123 |
+
const lower = Math.floor(index);
|
| 124 |
+
const upper = Math.ceil(index);
|
| 125 |
+
if (lower === upper) return sorted[lower];
|
| 126 |
+
return sorted[lower] + (sorted[upper] - sorted[lower]) * (index - lower);
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
async function run(): Promise<void> {
|
| 130 |
+
const backendName = textParameter("backend", "direct") as BackendName;
|
| 131 |
+
const provider = textParameter("provider", "wasm") as ProviderName;
|
| 132 |
+
if (!(["direct", "diffusion"] as string[]).includes(backendName)) throw new RangeError("backend must be direct or diffusion");
|
| 133 |
+
if (!(["wasm", "webgpu"] as string[]).includes(provider)) throw new RangeError("provider must be wasm or webgpu");
|
| 134 |
+
const sessions = parameter("sessions", 1);
|
| 135 |
+
const warmup = parameter("warmup", 1);
|
| 136 |
+
const repeats = parameter("repeats", 3);
|
| 137 |
+
const caseCount = parameter("cases", 20);
|
| 138 |
+
const runId = crypto.randomUUID();
|
| 139 |
+
const cases = Array.from({ length: caseCount }, (_, index) => ({
|
| 140 |
+
caseId: `case-${index + 1}`,
|
| 141 |
+
state: `browser benchmark state ${index + 1}`,
|
| 142 |
+
}));
|
| 143 |
+
const request = (caseIndex: number) => ({
|
| 144 |
+
state: cases[caseIndex % cases.length].state,
|
| 145 |
+
questions: [{ id: cases[caseIndex % cases.length].caseId, type: "choice" as const, prompt: "Choose the next action", options: ["continue", "stop", "inspect", "defer"] }],
|
| 146 |
+
seed: 7,
|
| 147 |
+
});
|
| 148 |
+
const inputHashes = new Map(cases.map((item, index) => [item.caseId, stableJson(request(index))]));
|
| 149 |
+
const hashedInputs = new Map<string, string>();
|
| 150 |
+
for (const [caseId, value] of inputHashes) hashedInputs.set(caseId, await sha256(value));
|
| 151 |
+
const noise = backendName === "diffusion" ? FROZEN_NOISE_SEED_7 : null;
|
| 152 |
+
const noiseHash = noise ? await sha256(noise.buffer) : null;
|
| 153 |
+
const browserEnvironment = {
|
| 154 |
+
user_agent: navigator.userAgent,
|
| 155 |
+
platform: navigator.platform || null,
|
| 156 |
+
hardware_concurrency: navigator.hardwareConcurrency ?? null,
|
| 157 |
+
device_memory_gb: (navigator as NavigatorWithDeviceMemory).deviceMemory ?? null,
|
| 158 |
+
cross_origin_isolated: globalThis.crossOriginIsolated,
|
| 159 |
+
};
|
| 160 |
+
const samples: BenchmarkSample[] = [];
|
| 161 |
+
const sessionRecords: SessionRecord[] = [];
|
| 162 |
+
let runtimeInfo: Awaited<ReturnType<typeof createOnnxWebBackend>>["runtimeInfo"] | null = null;
|
| 163 |
+
|
| 164 |
+
for (let sessionStart = 1; sessionStart <= sessions; sessionStart += 1) {
|
| 165 |
+
let timing: OnnxTimingSample | undefined;
|
| 166 |
+
const started = performance.now();
|
| 167 |
+
let backend: Awaited<ReturnType<typeof createOnnxWebBackend>> | undefined;
|
| 168 |
+
try {
|
| 169 |
+
backend = await createOnnxWebBackend({
|
| 170 |
+
manifestUrl: manifestUrls[backendName],
|
| 171 |
+
expectedManifestSha256: manifestHashes[backendName],
|
| 172 |
+
provider,
|
| 173 |
+
wasmPaths: "/sdk/typescript/node_modules/onnxruntime-web/dist/",
|
| 174 |
+
wasmNumThreads: 1,
|
| 175 |
+
onTiming: (sample) => { timing = sample; },
|
| 176 |
+
initialNoise: backendName === "diffusion" ? () => new Float32Array(noise as Float32Array) : undefined,
|
| 177 |
+
});
|
| 178 |
+
runtimeInfo = backend.runtimeInfo;
|
| 179 |
+
sessionRecords.push({ session_start: sessionStart, load_ms: performance.now() - started, status: "ok" });
|
| 180 |
+
for (let repeat = 0; repeat < warmup; repeat += 1) await backend.decide(request(repeat));
|
| 181 |
+
for (let repeat = 0; repeat < repeats; repeat += 1) {
|
| 182 |
+
const caseValue = cases[repeat % cases.length];
|
| 183 |
+
const before = heapBytes();
|
| 184 |
+
timing = undefined;
|
| 185 |
+
try {
|
| 186 |
+
await backend.decide(request(repeat));
|
| 187 |
+
if (!timing) throw new Error("timing hook did not produce a sample");
|
| 188 |
+
samples.push({
|
| 189 |
+
...timing,
|
| 190 |
+
run_id: runId,
|
| 191 |
+
session_start: sessionStart,
|
| 192 |
+
repeat,
|
| 193 |
+
case_id: caseValue.caseId,
|
| 194 |
+
backend: backendName,
|
| 195 |
+
requested_provider: provider,
|
| 196 |
+
observed_provider: null,
|
| 197 |
+
observed_provider_status: "session_created; execution_partition_not_exposed",
|
| 198 |
+
live_candidates: timing.live_candidates,
|
| 199 |
+
allocated_candidates: timing.allocated_candidates,
|
| 200 |
+
live_tokens: timing.live_tokens,
|
| 201 |
+
sequence_length: timing.sequence_length,
|
| 202 |
+
bundle_hash: backend.manifestHash,
|
| 203 |
+
tokenizer_hash: backend.manifest.files.find((item) => item.path.endsWith("tokenizer.json"))?.sha256 ?? null,
|
| 204 |
+
input_hash: hashedInputs.get(caseValue.caseId) as string,
|
| 205 |
+
noise_hash: noiseHash,
|
| 206 |
+
js_heap_used_bytes_before: before,
|
| 207 |
+
js_heap_used_bytes_after: heapBytes(),
|
| 208 |
+
status: "ok",
|
| 209 |
+
});
|
| 210 |
+
} catch (error) {
|
| 211 |
+
samples.push({
|
| 212 |
+
questionId: caseValue.caseId,
|
| 213 |
+
tokenizeMs: null,
|
| 214 |
+
prepareFeedMs: null,
|
| 215 |
+
inferenceAndReadbackMs: null,
|
| 216 |
+
postprocessMs: null,
|
| 217 |
+
totalRequestMs: null,
|
| 218 |
+
run_id: runId,
|
| 219 |
+
session_start: sessionStart,
|
| 220 |
+
repeat,
|
| 221 |
+
case_id: caseValue.caseId,
|
| 222 |
+
backend: backendName,
|
| 223 |
+
requested_provider: provider,
|
| 224 |
+
observed_provider: null,
|
| 225 |
+
observed_provider_status: "request_failed; execution_partition_not_exposed",
|
| 226 |
+
live_candidates: null,
|
| 227 |
+
allocated_candidates: null,
|
| 228 |
+
live_tokens: null,
|
| 229 |
+
sequence_length: null,
|
| 230 |
+
bundle_hash: backend.manifestHash,
|
| 231 |
+
tokenizer_hash: backend.manifest.files.find((item) => item.path.endsWith("tokenizer.json"))?.sha256 ?? null,
|
| 232 |
+
input_hash: hashedInputs.get(caseValue.caseId) as string,
|
| 233 |
+
noise_hash: noiseHash,
|
| 234 |
+
js_heap_used_bytes_before: before,
|
| 235 |
+
js_heap_used_bytes_after: heapBytes(),
|
| 236 |
+
status: "error",
|
| 237 |
+
error_kind: error instanceof Error ? `${error.name}: ${error.message}` : String(error),
|
| 238 |
+
});
|
| 239 |
+
}
|
| 240 |
+
}
|
| 241 |
+
} catch (error) {
|
| 242 |
+
sessionRecords.push({ session_start: sessionStart, load_ms: performance.now() - started, status: "error", error_kind: error instanceof Error ? `${error.name}: ${error.message}` : String(error) });
|
| 243 |
+
} finally {
|
| 244 |
+
await backend?.dispose();
|
| 245 |
+
}
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
const successful = samples.filter((sample) => sample.status === "ok");
|
| 249 |
+
const memoryMaximum = (values: Array<number | null>): number | null => {
|
| 250 |
+
const available = values.filter((value): value is number => value !== null);
|
| 251 |
+
return available.length === 0 ? null : Math.max(...available);
|
| 252 |
+
};
|
| 253 |
+
const report = {
|
| 254 |
+
schema: "vons.browser-benchmark/v1",
|
| 255 |
+
run_id: runId,
|
| 256 |
+
backend: backendName,
|
| 257 |
+
requested_provider: provider,
|
| 258 |
+
sessions,
|
| 259 |
+
warmup_excluded_per_session: warmup,
|
| 260 |
+
repeats_per_session: repeats,
|
| 261 |
+
cases: caseCount,
|
| 262 |
+
environment: browserEnvironment,
|
| 263 |
+
runtime_info: runtimeInfo,
|
| 264 |
+
session_start_semantics: "new ORT session on one page; module and browser caches may persist",
|
| 265 |
+
execution_provider_evidence: "requested provider and session creation only; graph partition status is not exposed by this harness",
|
| 266 |
+
session_records: sessionRecords,
|
| 267 |
+
samples,
|
| 268 |
+
summary: {
|
| 269 |
+
successful_samples: successful.length,
|
| 270 |
+
failed_samples: samples.length - successful.length,
|
| 271 |
+
total_request_ms_p50: percentile(successful.flatMap((sample) => sample.totalRequestMs === null ? [] : [sample.totalRequestMs]), 0.5),
|
| 272 |
+
total_request_ms_p95: percentile(successful.flatMap((sample) => sample.totalRequestMs === null ? [] : [sample.totalRequestMs]), 0.95),
|
| 273 |
+
inference_and_readback_ms_p50: percentile(successful.flatMap((sample) => sample.inferenceAndReadbackMs === null ? [] : [sample.inferenceAndReadbackMs]), 0.5),
|
| 274 |
+
inference_and_readback_ms_p95: percentile(successful.flatMap((sample) => sample.inferenceAndReadbackMs === null ? [] : [sample.inferenceAndReadbackMs]), 0.95),
|
| 275 |
+
js_heap_before_max: memoryMaximum(successful.map((sample) => sample.js_heap_used_bytes_before)),
|
| 276 |
+
js_heap_after_max: memoryMaximum(successful.map((sample) => sample.js_heap_used_bytes_after)),
|
| 277 |
+
memory_note: "performance.memory before/after observations only; not a peak or GPU-memory measurement",
|
| 278 |
+
},
|
| 279 |
+
};
|
| 280 |
+
showReport(report, `vons-${backendName}-${provider}-${runId}.json`);
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
void run().catch((error) => {
|
| 284 |
+
showReport({ schema: "vons.browser-benchmark/v1", status: "error", error: error instanceof Error ? `${error.name}: ${error.message}` : String(error) }, `vons-browser-benchmark-error-${crypto.randomUUID()}.json`);
|
| 285 |
+
});
|
sdk/typescript/demo/index.html
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8" />
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
| 6 |
+
<title>Vons ONNX Runtime Web smoke</title>
|
| 7 |
+
</head>
|
| 8 |
+
<body>
|
| 9 |
+
<main>
|
| 10 |
+
<h1>Vons ONNX Runtime Web smoke</h1>
|
| 11 |
+
<p>This harness is intentionally local-only. It reports runtime evidence; it is not a benchmark.</p>
|
| 12 |
+
<button data-backend="direct">Run Direct / WASM</button>
|
| 13 |
+
<button data-backend="diffusion">Run Diffusion / WASM</button>
|
| 14 |
+
<button data-backend="direct-webgpu">Try Direct / WebGPU</button>
|
| 15 |
+
<button data-backend="diffusion-webgpu">Try Diffusion / WebGPU</button>
|
| 16 |
+
<pre id="output">Not run.</pre>
|
| 17 |
+
</main>
|
| 18 |
+
<script type="module" src="./main.js"></script>
|
| 19 |
+
</body>
|
| 20 |
+
</html>
|
sdk/typescript/demo/main.ts
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { createOnnxWebBackend } from "../src/onnx.ts";
|
| 2 |
+
|
| 3 |
+
const outputElement = document.querySelector<HTMLElement>("#output");
|
| 4 |
+
if (!outputElement) throw new Error("smoke output element is missing");
|
| 5 |
+
const output = outputElement;
|
| 6 |
+
|
| 7 |
+
interface PerformanceWithMemory extends Performance {
|
| 8 |
+
memory?: { usedJSHeapSize: number };
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
const heapBytes = (): number | null => (performance as PerformanceWithMemory).memory?.usedJSHeapSize ?? null;
|
| 12 |
+
|
| 13 |
+
const manifestUrls = {
|
| 14 |
+
direct: new URL("../../../artifacts/pilot/bundle-manifest-v1.json", import.meta.url),
|
| 15 |
+
diffusion: new URL("../../../artifacts/pilot-diffusion/bundle-manifest-v1.json", import.meta.url),
|
| 16 |
+
};
|
| 17 |
+
|
| 18 |
+
const manifestHashes = {
|
| 19 |
+
direct: "bd39eea2b8d5ed5f4d57daefdd4822c4e2c4817a68c4a42bef850b5edcbc97d",
|
| 20 |
+
diffusion: "df0e7cae640c6604fb517a88d08dbdaa2454d6b3dbbf5708926349cf490b9607",
|
| 21 |
+
};
|
| 22 |
+
|
| 23 |
+
function show(value: unknown): void {
|
| 24 |
+
output.textContent = typeof value === "string" ? value : JSON.stringify(value, null, 2);
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
async function run(provider: "wasm" | "webgpu", backendName: "direct" | "diffusion"): Promise<void> {
|
| 28 |
+
const started = performance.now();
|
| 29 |
+
let backend: Awaited<ReturnType<typeof createOnnxWebBackend>> | undefined;
|
| 30 |
+
try {
|
| 31 |
+
backend = await createOnnxWebBackend({
|
| 32 |
+
manifestUrl: manifestUrls[backendName],
|
| 33 |
+
expectedManifestSha256: manifestHashes[backendName],
|
| 34 |
+
provider,
|
| 35 |
+
wasmPaths: "/sdk/typescript/node_modules/onnxruntime-web/dist/",
|
| 36 |
+
wasmNumThreads: 1,
|
| 37 |
+
});
|
| 38 |
+
const initialized = performance.now();
|
| 39 |
+
const beforeMemory = heapBytes();
|
| 40 |
+
const response = await backend.decide({
|
| 41 |
+
state: "browser smoke state",
|
| 42 |
+
questions: [{ id: "smoke", type: "choice", prompt: "Choose the next action", options: ["continue", "stop"] }],
|
| 43 |
+
seed: 7,
|
| 44 |
+
});
|
| 45 |
+
const completed = performance.now();
|
| 46 |
+
show({
|
| 47 |
+
pass: true,
|
| 48 |
+
backend: backendName,
|
| 49 |
+
runtime: backend.runtimeInfo,
|
| 50 |
+
load_ms: initialized - started,
|
| 51 |
+
inference_ms: completed - initialized,
|
| 52 |
+
total_ms: completed - started,
|
| 53 |
+
js_heap_used_bytes_before: beforeMemory,
|
| 54 |
+
js_heap_used_bytes_after: heapBytes(),
|
| 55 |
+
response,
|
| 56 |
+
});
|
| 57 |
+
} catch (error) {
|
| 58 |
+
show({
|
| 59 |
+
pass: false,
|
| 60 |
+
backend: backendName,
|
| 61 |
+
provider,
|
| 62 |
+
elapsed_ms: performance.now() - started,
|
| 63 |
+
error: error instanceof Error ? `${error.name}: ${error.message}` : String(error),
|
| 64 |
+
});
|
| 65 |
+
} finally {
|
| 66 |
+
await backend?.dispose();
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
for (const button of Array.from(document.querySelectorAll<HTMLButtonElement>("button[data-backend]"))) {
|
| 71 |
+
button.addEventListener("click", () => {
|
| 72 |
+
const value = button.dataset.backend;
|
| 73 |
+
if (value === "direct") void run("wasm", "direct");
|
| 74 |
+
else if (value === "diffusion") void run("wasm", "diffusion");
|
| 75 |
+
else if (value === "direct-webgpu") void run("webgpu", "direct");
|
| 76 |
+
else if (value === "diffusion-webgpu") void run("webgpu", "diffusion");
|
| 77 |
+
});
|
| 78 |
+
}
|
sdk/typescript/demo/parity.html
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8" />
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
| 6 |
+
<title>Vons ONNX Runtime Web parity matrix</title>
|
| 7 |
+
</head>
|
| 8 |
+
<body>
|
| 9 |
+
<main>
|
| 10 |
+
<h1>Vons ONNX Runtime Web parity matrix</h1>
|
| 11 |
+
<p>This local harness runs the frozen 23-fixture contract matrix. It is not a quality benchmark.</p>
|
| 12 |
+
<button id="run">Run selected backend/provider</button>
|
| 13 |
+
<button id="download" disabled>Download report</button>
|
| 14 |
+
<label>Backend <select id="backend"><option value="direct">Direct</option><option value="diffusion">Diffusion</option></select></label>
|
| 15 |
+
<label>Provider <select id="provider"><option value="wasm">WASM</option><option value="webgpu">WebGPU</option></select></label>
|
| 16 |
+
<pre id="output">Not run.</pre>
|
| 17 |
+
</main>
|
| 18 |
+
<script type="module" src="./parity.js"></script>
|
| 19 |
+
</body>
|
| 20 |
+
</html>
|
sdk/typescript/demo/parity.ts
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { createOnnxWebBackend, type OnnxTimingSample, type OnnxTraceSample } from "../src/onnx.ts";
|
| 2 |
+
import type { DecisionRequest, DecisionResponse, QuestionType } from "../src/index.ts";
|
| 3 |
+
|
| 4 |
+
export type ParityBackend = "direct" | "diffusion";
|
| 5 |
+
export type ParityProvider = "wasm" | "webgpu";
|
| 6 |
+
|
| 7 |
+
export interface ParityFixture {
|
| 8 |
+
id: string;
|
| 9 |
+
request: DecisionRequest;
|
| 10 |
+
expectedFailure?: string;
|
| 11 |
+
matrix: { candidate_count: number; family: string; target_tokens: string };
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
export interface ParityRecord {
|
| 15 |
+
fixture_id: string;
|
| 16 |
+
input_hash: string;
|
| 17 |
+
request: DecisionRequest;
|
| 18 |
+
matrix: ParityFixture["matrix"];
|
| 19 |
+
status: "ok" | "expected_error" | "unexpected_error";
|
| 20 |
+
response?: DecisionResponse;
|
| 21 |
+
timings?: OnnxTimingSample[];
|
| 22 |
+
traces?: OnnxTraceSample[];
|
| 23 |
+
error?: string;
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
const manifestUrls: Record<ParityBackend, URL> = {
|
| 27 |
+
direct: new URL("../../../artifacts/pilot/bundle-manifest-v1.json", import.meta.url),
|
| 28 |
+
diffusion: new URL("../../../artifacts/pilot-diffusion/bundle-manifest-v1.json", import.meta.url),
|
| 29 |
+
};
|
| 30 |
+
|
| 31 |
+
const manifestHashes: Record<ParityBackend, string> = {
|
| 32 |
+
direct: "bd39eea2b8d5ed5f4d57daefdd4822c4e2c4817a68c4a42bef850b5edcbc97d",
|
| 33 |
+
diffusion: "df0e7cae640c6604fb517a88d08dbdaa2454d6b3dbbf5708926349cf490b9607",
|
| 34 |
+
};
|
| 35 |
+
|
| 36 |
+
const FROZEN_NOISE_SEED_7 = Float32Array.from([
|
| 37 |
+
1.52196932, -1.14410579, 1.15016162, -1.78465319, -0.77149874, 1.01916146,
|
| 38 |
+
-0.926134586, -0.415745318, -0.0629121512, 0.513251185, -0.285995334, 1.98480856,
|
| 39 |
+
1.30306911, -0.00464721443, 1.17548203, 1.34566987, -0.171809882, -2.4133966,
|
| 40 |
+
-0.0993812904, -0.979565322, 0.942675471, 0.452982962, 0.53610903, 0.494613141,
|
| 41 |
+
1.94684041, 0.207542002, -1.64390635, -0.542155981, -0.384397745, -0.410856962,
|
| 42 |
+
0.946411192, 0.979099393,
|
| 43 |
+
]);
|
| 44 |
+
|
| 45 |
+
function options(count: number, label: string): string[] {
|
| 46 |
+
return Array.from({ length: count }, (_, index) => `${label} candidate ${index + 1}`);
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
function question(
|
| 50 |
+
id: string,
|
| 51 |
+
type: QuestionType,
|
| 52 |
+
prompt: string,
|
| 53 |
+
values: string[],
|
| 54 |
+
): DecisionRequest["questions"][number] {
|
| 55 |
+
return type === "score"
|
| 56 |
+
? { id, type, prompt, options: values, rubric: ["poor", "fair", "good", "very good", "excellent"] }
|
| 57 |
+
: { id, type, prompt, options: values };
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
function fixture(
|
| 61 |
+
id: string,
|
| 62 |
+
family: string,
|
| 63 |
+
targetTokens: string,
|
| 64 |
+
candidateCount: number,
|
| 65 |
+
state: DecisionRequest["state"],
|
| 66 |
+
item: DecisionRequest["questions"][number],
|
| 67 |
+
expectedFailure?: string,
|
| 68 |
+
): ParityFixture {
|
| 69 |
+
return {
|
| 70 |
+
id,
|
| 71 |
+
request: { state, questions: [item], seed: 7 },
|
| 72 |
+
expectedFailure,
|
| 73 |
+
matrix: { candidate_count: candidateCount, family, target_tokens: targetTokens },
|
| 74 |
+
};
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
export function buildParityFixtures(): ParityFixture[] {
|
| 78 |
+
const result: ParityFixture[] = [];
|
| 79 |
+
const counts = [2, 4, 8, 16, 32];
|
| 80 |
+
for (const count of counts) {
|
| 81 |
+
result.push(fixture(`short-${count}`, "choice", "short", count, "ready", question(`short-${count}-q`, "choice", "Choose the next action", options(count, "short"))));
|
| 82 |
+
result.push(fixture(`unicode-${count}`, "choice", "unicode", count, { locale: "ko-KR", state: `검토 중인 화면 🌐 / пункт ${count}` }, question(`unicode-${count}-q`, "choice", "다음에 선택할 항목은 무엇입니까?", options(count, "unicode/다음"))));
|
| 83 |
+
result.push(fixture(`score-${count}`, "score", "unsupported", count, { workflow: "quality", count, labels: ["α", "β", "γ"] }, question(`score-${count}-q`, "score", "Rate the candidate outcome", options(count, "score")), "score_head_unsupported"));
|
| 84 |
+
const repeats = Math.max(2, Math.floor(60 / count));
|
| 85 |
+
result.push(fixture(`long-${count}`, "choice", "near-aggregate-limit", count, "state ".repeat(50), question(`long-${count}-q`, "choice", "Select the most relevant detailed action", options(count, "detail ".repeat(repeats)))));
|
| 86 |
+
}
|
| 87 |
+
result.push(fixture("boolean-2", "boolean", "boolean", 2, "permission check", question("boolean-q", "boolean", "Should the host ask for confirmation?", [])));
|
| 88 |
+
result.push({
|
| 89 |
+
id: "multi-question",
|
| 90 |
+
request: {
|
| 91 |
+
state: { page: "settings", text: "Review before any external action" },
|
| 92 |
+
questions: [
|
| 93 |
+
question("multi-choice", "choice", "Which next step is safest?", options(4, "multi")),
|
| 94 |
+
question("multi-boolean", "boolean", "Is confirmation required?", []),
|
| 95 |
+
],
|
| 96 |
+
seed: 7,
|
| 97 |
+
},
|
| 98 |
+
matrix: { candidate_count: 4, family: "multi-question", target_tokens: "mixed" },
|
| 99 |
+
});
|
| 100 |
+
result.push(fixture(
|
| 101 |
+
"overflow-explicit",
|
| 102 |
+
"overflow",
|
| 103 |
+
"over-limit",
|
| 104 |
+
2,
|
| 105 |
+
"x ".repeat(700),
|
| 106 |
+
question("overflow-q", "choice", "This request must fail explicitly", options(2, "overflow")),
|
| 107 |
+
"request_validation_or_tokenization_overflow",
|
| 108 |
+
));
|
| 109 |
+
return result;
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
function stableJson(value: unknown): string {
|
| 113 |
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
| 114 |
+
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
| 115 |
+
const entries = Object.entries(value as Record<string, unknown>).sort(([a], [b]) => a.localeCompare(b));
|
| 116 |
+
return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(",")}}`;
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
async function sha256(value: string): Promise<string> {
|
| 120 |
+
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value));
|
| 121 |
+
return Array.from(new Uint8Array(digest), (item) => item.toString(16).padStart(2, "0")).join("");
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
export async function runParityCell(backendName: ParityBackend, provider: ParityProvider): Promise<Record<string, unknown>> {
|
| 125 |
+
const fixtures = buildParityFixtures();
|
| 126 |
+
const records: ParityRecord[] = [];
|
| 127 |
+
let timings: OnnxTimingSample[] = [];
|
| 128 |
+
let traces: OnnxTraceSample[] = [];
|
| 129 |
+
const backend = await createOnnxWebBackend({
|
| 130 |
+
manifestUrl: manifestUrls[backendName],
|
| 131 |
+
expectedManifestSha256: manifestHashes[backendName],
|
| 132 |
+
provider,
|
| 133 |
+
wasmPaths: "/sdk/typescript/node_modules/onnxruntime-web/dist/",
|
| 134 |
+
wasmNumThreads: 1,
|
| 135 |
+
initialNoise: backendName === "diffusion" ? () => new Float32Array(FROZEN_NOISE_SEED_7) : undefined,
|
| 136 |
+
onTiming: (sample) => { timings.push(sample); },
|
| 137 |
+
onTrace: (sample) => { traces.push(sample); },
|
| 138 |
+
});
|
| 139 |
+
try {
|
| 140 |
+
for (const item of fixtures) {
|
| 141 |
+
const inputHash = await sha256(stableJson(item.request));
|
| 142 |
+
timings = [];
|
| 143 |
+
traces = [];
|
| 144 |
+
try {
|
| 145 |
+
const response = await backend.decide(item.request);
|
| 146 |
+
records.push({ fixture_id: item.id, input_hash: inputHash, request: item.request, matrix: item.matrix, status: "ok", response, timings, traces });
|
| 147 |
+
} catch (error) {
|
| 148 |
+
const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
| 149 |
+
records.push({
|
| 150 |
+
fixture_id: item.id,
|
| 151 |
+
input_hash: inputHash,
|
| 152 |
+
request: item.request,
|
| 153 |
+
matrix: item.matrix,
|
| 154 |
+
status: item.expectedFailure ? "expected_error" : "unexpected_error",
|
| 155 |
+
timings,
|
| 156 |
+
traces,
|
| 157 |
+
error: message,
|
| 158 |
+
});
|
| 159 |
+
}
|
| 160 |
+
}
|
| 161 |
+
} finally {
|
| 162 |
+
await backend.dispose();
|
| 163 |
+
}
|
| 164 |
+
return {
|
| 165 |
+
schema: "vons.browser-parity/v1",
|
| 166 |
+
backend: backendName,
|
| 167 |
+
requested_provider: provider,
|
| 168 |
+
fixture_count: fixtures.length,
|
| 169 |
+
runtime_info: backend.runtimeInfo,
|
| 170 |
+
manifest_hash: backend.manifestHash,
|
| 171 |
+
tokenizer_hash: backend.manifest.files.find((item) => item.path.endsWith("tokenizer.json"))?.sha256 ?? null,
|
| 172 |
+
noise_seed: backendName === "diffusion" ? 7 : null,
|
| 173 |
+
noise_values: backendName === "diffusion" ? Array.from(FROZEN_NOISE_SEED_7) : null,
|
| 174 |
+
records,
|
| 175 |
+
limitations: [
|
| 176 |
+
"This harness checks contract-valid responses and reproducible fixture inputs; it does not establish task accuracy.",
|
| 177 |
+
"A browser-vs-Python numeric parity claim requires separately saved CPU reference outputs for the same input and noise hashes.",
|
| 178 |
+
"WebGPU provider selection does not expose graph partition evidence in this runtime harness.",
|
| 179 |
+
],
|
| 180 |
+
};
|
| 181 |
+
}
|
| 182 |
+
|
| 183 |
+
if (typeof document !== "undefined") {
|
| 184 |
+
const runButton = document.querySelector<HTMLButtonElement>("#run");
|
| 185 |
+
const output = document.querySelector<HTMLElement>("#output");
|
| 186 |
+
const backendSelect = document.querySelector<HTMLSelectElement>("#backend");
|
| 187 |
+
const providerSelect = document.querySelector<HTMLSelectElement>("#provider");
|
| 188 |
+
const downloadButton = document.querySelector<HTMLButtonElement>("#download");
|
| 189 |
+
if (!runButton || !output || !backendSelect || !providerSelect || !downloadButton) throw new Error("parity harness controls are missing");
|
| 190 |
+
let lastReport: Record<string, unknown> | null = null;
|
| 191 |
+
const displayReport = (report: Record<string, unknown>): void => {
|
| 192 |
+
const records = Array.isArray(report.records) ? report.records as Array<Record<string, unknown>> : [];
|
| 193 |
+
const statusCounts = records.reduce<Record<string, number>>((counts, record) => {
|
| 194 |
+
const status = String(record.status ?? "unknown");
|
| 195 |
+
counts[status] = (counts[status] ?? 0) + 1;
|
| 196 |
+
return counts;
|
| 197 |
+
}, {});
|
| 198 |
+
output.textContent = JSON.stringify({
|
| 199 |
+
schema: report.schema,
|
| 200 |
+
backend: report.backend,
|
| 201 |
+
requested_provider: report.requested_provider,
|
| 202 |
+
fixture_count: report.fixture_count,
|
| 203 |
+
manifest_hash: report.manifest_hash,
|
| 204 |
+
tokenizer_hash: report.tokenizer_hash,
|
| 205 |
+
noise_seed: report.noise_seed,
|
| 206 |
+
status_counts: statusCounts,
|
| 207 |
+
errors: records.filter((record) => record.status !== "ok").map((record) => ({ fixture_id: record.fixture_id, status: record.status, error: record.error })),
|
| 208 |
+
note: "Full parity JSON, including request hashes, timings, traces, and noise values, is available through Download report.",
|
| 209 |
+
}, null, 2);
|
| 210 |
+
};
|
| 211 |
+
downloadButton.addEventListener("click", () => {
|
| 212 |
+
if (!lastReport) return;
|
| 213 |
+
const backend = backendSelect.value as ParityBackend;
|
| 214 |
+
const provider = providerSelect.value as ParityProvider;
|
| 215 |
+
const blob = new Blob([JSON.stringify(lastReport, null, 2)], { type: "application/json" });
|
| 216 |
+
const url = URL.createObjectURL(blob);
|
| 217 |
+
const link = document.createElement("a");
|
| 218 |
+
link.href = url;
|
| 219 |
+
link.download = `vons-parity-${backend}-${provider}.json`;
|
| 220 |
+
link.click();
|
| 221 |
+
URL.revokeObjectURL(url);
|
| 222 |
+
});
|
| 223 |
+
runButton.addEventListener("click", async () => {
|
| 224 |
+
runButton.disabled = true;
|
| 225 |
+
downloadButton.disabled = true;
|
| 226 |
+
lastReport = null;
|
| 227 |
+
output.textContent = "Running...";
|
| 228 |
+
try {
|
| 229 |
+
const report = await runParityCell(backendSelect.value as ParityBackend, providerSelect.value as ParityProvider);
|
| 230 |
+
lastReport = report;
|
| 231 |
+
downloadButton.disabled = false;
|
| 232 |
+
displayReport(report);
|
| 233 |
+
} catch (error) {
|
| 234 |
+
output.textContent = JSON.stringify({ schema: "vons.browser-parity/v1", status: "error", error: String(error) }, null, 2);
|
| 235 |
+
} finally {
|
| 236 |
+
runButton.disabled = false;
|
| 237 |
+
}
|
| 238 |
+
});
|
| 239 |
+
}
|
sdk/typescript/package-lock.json
ADDED
|
@@ -0,0 +1,652 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "@vons/decision-sdk",
|
| 3 |
+
"version": "0.1.0",
|
| 4 |
+
"lockfileVersion": 3,
|
| 5 |
+
"requires": true,
|
| 6 |
+
"packages": {
|
| 7 |
+
"": {
|
| 8 |
+
"name": "@vons/decision-sdk",
|
| 9 |
+
"version": "0.1.0",
|
| 10 |
+
"dependencies": {
|
| 11 |
+
"@huggingface/tokenizers": "0.2.0",
|
| 12 |
+
"onnxruntime-web": "1.30.0"
|
| 13 |
+
},
|
| 14 |
+
"devDependencies": {
|
| 15 |
+
"esbuild": "0.28.2"
|
| 16 |
+
},
|
| 17 |
+
"engines": {
|
| 18 |
+
"node": ">=22"
|
| 19 |
+
},
|
| 20 |
+
"license": "SEE LICENSE IN LICENSE"
|
| 21 |
+
},
|
| 22 |
+
"node_modules/@esbuild/aix-ppc64": {
|
| 23 |
+
"version": "0.28.2",
|
| 24 |
+
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz",
|
| 25 |
+
"integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==",
|
| 26 |
+
"cpu": [
|
| 27 |
+
"ppc64"
|
| 28 |
+
],
|
| 29 |
+
"dev": true,
|
| 30 |
+
"license": "MIT",
|
| 31 |
+
"optional": true,
|
| 32 |
+
"os": [
|
| 33 |
+
"aix"
|
| 34 |
+
],
|
| 35 |
+
"engines": {
|
| 36 |
+
"node": ">=18"
|
| 37 |
+
}
|
| 38 |
+
},
|
| 39 |
+
"node_modules/@esbuild/android-arm": {
|
| 40 |
+
"version": "0.28.2",
|
| 41 |
+
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz",
|
| 42 |
+
"integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==",
|
| 43 |
+
"cpu": [
|
| 44 |
+
"arm"
|
| 45 |
+
],
|
| 46 |
+
"dev": true,
|
| 47 |
+
"license": "MIT",
|
| 48 |
+
"optional": true,
|
| 49 |
+
"os": [
|
| 50 |
+
"android"
|
| 51 |
+
],
|
| 52 |
+
"engines": {
|
| 53 |
+
"node": ">=18"
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
"node_modules/@esbuild/android-arm64": {
|
| 57 |
+
"version": "0.28.2",
|
| 58 |
+
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz",
|
| 59 |
+
"integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==",
|
| 60 |
+
"cpu": [
|
| 61 |
+
"arm64"
|
| 62 |
+
],
|
| 63 |
+
"dev": true,
|
| 64 |
+
"license": "MIT",
|
| 65 |
+
"optional": true,
|
| 66 |
+
"os": [
|
| 67 |
+
"android"
|
| 68 |
+
],
|
| 69 |
+
"engines": {
|
| 70 |
+
"node": ">=18"
|
| 71 |
+
}
|
| 72 |
+
},
|
| 73 |
+
"node_modules/@esbuild/android-x64": {
|
| 74 |
+
"version": "0.28.2",
|
| 75 |
+
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz",
|
| 76 |
+
"integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==",
|
| 77 |
+
"cpu": [
|
| 78 |
+
"x64"
|
| 79 |
+
],
|
| 80 |
+
"dev": true,
|
| 81 |
+
"license": "MIT",
|
| 82 |
+
"optional": true,
|
| 83 |
+
"os": [
|
| 84 |
+
"android"
|
| 85 |
+
],
|
| 86 |
+
"engines": {
|
| 87 |
+
"node": ">=18"
|
| 88 |
+
}
|
| 89 |
+
},
|
| 90 |
+
"node_modules/@esbuild/darwin-arm64": {
|
| 91 |
+
"version": "0.28.2",
|
| 92 |
+
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz",
|
| 93 |
+
"integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==",
|
| 94 |
+
"cpu": [
|
| 95 |
+
"arm64"
|
| 96 |
+
],
|
| 97 |
+
"dev": true,
|
| 98 |
+
"license": "MIT",
|
| 99 |
+
"optional": true,
|
| 100 |
+
"os": [
|
| 101 |
+
"darwin"
|
| 102 |
+
],
|
| 103 |
+
"engines": {
|
| 104 |
+
"node": ">=18"
|
| 105 |
+
}
|
| 106 |
+
},
|
| 107 |
+
"node_modules/@esbuild/darwin-x64": {
|
| 108 |
+
"version": "0.28.2",
|
| 109 |
+
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz",
|
| 110 |
+
"integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==",
|
| 111 |
+
"cpu": [
|
| 112 |
+
"x64"
|
| 113 |
+
],
|
| 114 |
+
"dev": true,
|
| 115 |
+
"license": "MIT",
|
| 116 |
+
"optional": true,
|
| 117 |
+
"os": [
|
| 118 |
+
"darwin"
|
| 119 |
+
],
|
| 120 |
+
"engines": {
|
| 121 |
+
"node": ">=18"
|
| 122 |
+
}
|
| 123 |
+
},
|
| 124 |
+
"node_modules/@esbuild/freebsd-arm64": {
|
| 125 |
+
"version": "0.28.2",
|
| 126 |
+
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz",
|
| 127 |
+
"integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==",
|
| 128 |
+
"cpu": [
|
| 129 |
+
"arm64"
|
| 130 |
+
],
|
| 131 |
+
"dev": true,
|
| 132 |
+
"license": "MIT",
|
| 133 |
+
"optional": true,
|
| 134 |
+
"os": [
|
| 135 |
+
"freebsd"
|
| 136 |
+
],
|
| 137 |
+
"engines": {
|
| 138 |
+
"node": ">=18"
|
| 139 |
+
}
|
| 140 |
+
},
|
| 141 |
+
"node_modules/@esbuild/freebsd-x64": {
|
| 142 |
+
"version": "0.28.2",
|
| 143 |
+
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz",
|
| 144 |
+
"integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==",
|
| 145 |
+
"cpu": [
|
| 146 |
+
"x64"
|
| 147 |
+
],
|
| 148 |
+
"dev": true,
|
| 149 |
+
"license": "MIT",
|
| 150 |
+
"optional": true,
|
| 151 |
+
"os": [
|
| 152 |
+
"freebsd"
|
| 153 |
+
],
|
| 154 |
+
"engines": {
|
| 155 |
+
"node": ">=18"
|
| 156 |
+
}
|
| 157 |
+
},
|
| 158 |
+
"node_modules/@esbuild/linux-arm": {
|
| 159 |
+
"version": "0.28.2",
|
| 160 |
+
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz",
|
| 161 |
+
"integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==",
|
| 162 |
+
"cpu": [
|
| 163 |
+
"arm"
|
| 164 |
+
],
|
| 165 |
+
"dev": true,
|
| 166 |
+
"license": "MIT",
|
| 167 |
+
"optional": true,
|
| 168 |
+
"os": [
|
| 169 |
+
"linux"
|
| 170 |
+
],
|
| 171 |
+
"engines": {
|
| 172 |
+
"node": ">=18"
|
| 173 |
+
}
|
| 174 |
+
},
|
| 175 |
+
"node_modules/@esbuild/linux-arm64": {
|
| 176 |
+
"version": "0.28.2",
|
| 177 |
+
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz",
|
| 178 |
+
"integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==",
|
| 179 |
+
"cpu": [
|
| 180 |
+
"arm64"
|
| 181 |
+
],
|
| 182 |
+
"dev": true,
|
| 183 |
+
"license": "MIT",
|
| 184 |
+
"optional": true,
|
| 185 |
+
"os": [
|
| 186 |
+
"linux"
|
| 187 |
+
],
|
| 188 |
+
"engines": {
|
| 189 |
+
"node": ">=18"
|
| 190 |
+
}
|
| 191 |
+
},
|
| 192 |
+
"node_modules/@esbuild/linux-ia32": {
|
| 193 |
+
"version": "0.28.2",
|
| 194 |
+
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz",
|
| 195 |
+
"integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==",
|
| 196 |
+
"cpu": [
|
| 197 |
+
"ia32"
|
| 198 |
+
],
|
| 199 |
+
"dev": true,
|
| 200 |
+
"license": "MIT",
|
| 201 |
+
"optional": true,
|
| 202 |
+
"os": [
|
| 203 |
+
"linux"
|
| 204 |
+
],
|
| 205 |
+
"engines": {
|
| 206 |
+
"node": ">=18"
|
| 207 |
+
}
|
| 208 |
+
},
|
| 209 |
+
"node_modules/@esbuild/linux-loong64": {
|
| 210 |
+
"version": "0.28.2",
|
| 211 |
+
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz",
|
| 212 |
+
"integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==",
|
| 213 |
+
"cpu": [
|
| 214 |
+
"loong64"
|
| 215 |
+
],
|
| 216 |
+
"dev": true,
|
| 217 |
+
"license": "MIT",
|
| 218 |
+
"optional": true,
|
| 219 |
+
"os": [
|
| 220 |
+
"linux"
|
| 221 |
+
],
|
| 222 |
+
"engines": {
|
| 223 |
+
"node": ">=18"
|
| 224 |
+
}
|
| 225 |
+
},
|
| 226 |
+
"node_modules/@esbuild/linux-mips64el": {
|
| 227 |
+
"version": "0.28.2",
|
| 228 |
+
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz",
|
| 229 |
+
"integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==",
|
| 230 |
+
"cpu": [
|
| 231 |
+
"mips64el"
|
| 232 |
+
],
|
| 233 |
+
"dev": true,
|
| 234 |
+
"license": "MIT",
|
| 235 |
+
"optional": true,
|
| 236 |
+
"os": [
|
| 237 |
+
"linux"
|
| 238 |
+
],
|
| 239 |
+
"engines": {
|
| 240 |
+
"node": ">=18"
|
| 241 |
+
}
|
| 242 |
+
},
|
| 243 |
+
"node_modules/@esbuild/linux-ppc64": {
|
| 244 |
+
"version": "0.28.2",
|
| 245 |
+
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz",
|
| 246 |
+
"integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==",
|
| 247 |
+
"cpu": [
|
| 248 |
+
"ppc64"
|
| 249 |
+
],
|
| 250 |
+
"dev": true,
|
| 251 |
+
"license": "MIT",
|
| 252 |
+
"optional": true,
|
| 253 |
+
"os": [
|
| 254 |
+
"linux"
|
| 255 |
+
],
|
| 256 |
+
"engines": {
|
| 257 |
+
"node": ">=18"
|
| 258 |
+
}
|
| 259 |
+
},
|
| 260 |
+
"node_modules/@esbuild/linux-riscv64": {
|
| 261 |
+
"version": "0.28.2",
|
| 262 |
+
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz",
|
| 263 |
+
"integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==",
|
| 264 |
+
"cpu": [
|
| 265 |
+
"riscv64"
|
| 266 |
+
],
|
| 267 |
+
"dev": true,
|
| 268 |
+
"license": "MIT",
|
| 269 |
+
"optional": true,
|
| 270 |
+
"os": [
|
| 271 |
+
"linux"
|
| 272 |
+
],
|
| 273 |
+
"engines": {
|
| 274 |
+
"node": ">=18"
|
| 275 |
+
}
|
| 276 |
+
},
|
| 277 |
+
"node_modules/@esbuild/linux-s390x": {
|
| 278 |
+
"version": "0.28.2",
|
| 279 |
+
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz",
|
| 280 |
+
"integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==",
|
| 281 |
+
"cpu": [
|
| 282 |
+
"s390x"
|
| 283 |
+
],
|
| 284 |
+
"dev": true,
|
| 285 |
+
"license": "MIT",
|
| 286 |
+
"optional": true,
|
| 287 |
+
"os": [
|
| 288 |
+
"linux"
|
| 289 |
+
],
|
| 290 |
+
"engines": {
|
| 291 |
+
"node": ">=18"
|
| 292 |
+
}
|
| 293 |
+
},
|
| 294 |
+
"node_modules/@esbuild/linux-x64": {
|
| 295 |
+
"version": "0.28.2",
|
| 296 |
+
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz",
|
| 297 |
+
"integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==",
|
| 298 |
+
"cpu": [
|
| 299 |
+
"x64"
|
| 300 |
+
],
|
| 301 |
+
"dev": true,
|
| 302 |
+
"license": "MIT",
|
| 303 |
+
"optional": true,
|
| 304 |
+
"os": [
|
| 305 |
+
"linux"
|
| 306 |
+
],
|
| 307 |
+
"engines": {
|
| 308 |
+
"node": ">=18"
|
| 309 |
+
}
|
| 310 |
+
},
|
| 311 |
+
"node_modules/@esbuild/netbsd-arm64": {
|
| 312 |
+
"version": "0.28.2",
|
| 313 |
+
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz",
|
| 314 |
+
"integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==",
|
| 315 |
+
"cpu": [
|
| 316 |
+
"arm64"
|
| 317 |
+
],
|
| 318 |
+
"dev": true,
|
| 319 |
+
"license": "MIT",
|
| 320 |
+
"optional": true,
|
| 321 |
+
"os": [
|
| 322 |
+
"netbsd"
|
| 323 |
+
],
|
| 324 |
+
"engines": {
|
| 325 |
+
"node": ">=18"
|
| 326 |
+
}
|
| 327 |
+
},
|
| 328 |
+
"node_modules/@esbuild/netbsd-x64": {
|
| 329 |
+
"version": "0.28.2",
|
| 330 |
+
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz",
|
| 331 |
+
"integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==",
|
| 332 |
+
"cpu": [
|
| 333 |
+
"x64"
|
| 334 |
+
],
|
| 335 |
+
"dev": true,
|
| 336 |
+
"license": "MIT",
|
| 337 |
+
"optional": true,
|
| 338 |
+
"os": [
|
| 339 |
+
"netbsd"
|
| 340 |
+
],
|
| 341 |
+
"engines": {
|
| 342 |
+
"node": ">=18"
|
| 343 |
+
}
|
| 344 |
+
},
|
| 345 |
+
"node_modules/@esbuild/openbsd-arm64": {
|
| 346 |
+
"version": "0.28.2",
|
| 347 |
+
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz",
|
| 348 |
+
"integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==",
|
| 349 |
+
"cpu": [
|
| 350 |
+
"arm64"
|
| 351 |
+
],
|
| 352 |
+
"dev": true,
|
| 353 |
+
"license": "MIT",
|
| 354 |
+
"optional": true,
|
| 355 |
+
"os": [
|
| 356 |
+
"openbsd"
|
| 357 |
+
],
|
| 358 |
+
"engines": {
|
| 359 |
+
"node": ">=18"
|
| 360 |
+
}
|
| 361 |
+
},
|
| 362 |
+
"node_modules/@esbuild/openbsd-x64": {
|
| 363 |
+
"version": "0.28.2",
|
| 364 |
+
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz",
|
| 365 |
+
"integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==",
|
| 366 |
+
"cpu": [
|
| 367 |
+
"x64"
|
| 368 |
+
],
|
| 369 |
+
"dev": true,
|
| 370 |
+
"license": "MIT",
|
| 371 |
+
"optional": true,
|
| 372 |
+
"os": [
|
| 373 |
+
"openbsd"
|
| 374 |
+
],
|
| 375 |
+
"engines": {
|
| 376 |
+
"node": ">=18"
|
| 377 |
+
}
|
| 378 |
+
},
|
| 379 |
+
"node_modules/@esbuild/openharmony-arm64": {
|
| 380 |
+
"version": "0.28.2",
|
| 381 |
+
"resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz",
|
| 382 |
+
"integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==",
|
| 383 |
+
"cpu": [
|
| 384 |
+
"arm64"
|
| 385 |
+
],
|
| 386 |
+
"dev": true,
|
| 387 |
+
"license": "MIT",
|
| 388 |
+
"optional": true,
|
| 389 |
+
"os": [
|
| 390 |
+
"openharmony"
|
| 391 |
+
],
|
| 392 |
+
"engines": {
|
| 393 |
+
"node": ">=18"
|
| 394 |
+
}
|
| 395 |
+
},
|
| 396 |
+
"node_modules/@esbuild/sunos-x64": {
|
| 397 |
+
"version": "0.28.2",
|
| 398 |
+
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz",
|
| 399 |
+
"integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==",
|
| 400 |
+
"cpu": [
|
| 401 |
+
"x64"
|
| 402 |
+
],
|
| 403 |
+
"dev": true,
|
| 404 |
+
"license": "MIT",
|
| 405 |
+
"optional": true,
|
| 406 |
+
"os": [
|
| 407 |
+
"sunos"
|
| 408 |
+
],
|
| 409 |
+
"engines": {
|
| 410 |
+
"node": ">=18"
|
| 411 |
+
}
|
| 412 |
+
},
|
| 413 |
+
"node_modules/@esbuild/win32-arm64": {
|
| 414 |
+
"version": "0.28.2",
|
| 415 |
+
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz",
|
| 416 |
+
"integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==",
|
| 417 |
+
"cpu": [
|
| 418 |
+
"arm64"
|
| 419 |
+
],
|
| 420 |
+
"dev": true,
|
| 421 |
+
"license": "MIT",
|
| 422 |
+
"optional": true,
|
| 423 |
+
"os": [
|
| 424 |
+
"win32"
|
| 425 |
+
],
|
| 426 |
+
"engines": {
|
| 427 |
+
"node": ">=18"
|
| 428 |
+
}
|
| 429 |
+
},
|
| 430 |
+
"node_modules/@esbuild/win32-ia32": {
|
| 431 |
+
"version": "0.28.2",
|
| 432 |
+
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz",
|
| 433 |
+
"integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==",
|
| 434 |
+
"cpu": [
|
| 435 |
+
"ia32"
|
| 436 |
+
],
|
| 437 |
+
"dev": true,
|
| 438 |
+
"license": "MIT",
|
| 439 |
+
"optional": true,
|
| 440 |
+
"os": [
|
| 441 |
+
"win32"
|
| 442 |
+
],
|
| 443 |
+
"engines": {
|
| 444 |
+
"node": ">=18"
|
| 445 |
+
}
|
| 446 |
+
},
|
| 447 |
+
"node_modules/@esbuild/win32-x64": {
|
| 448 |
+
"version": "0.28.2",
|
| 449 |
+
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz",
|
| 450 |
+
"integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==",
|
| 451 |
+
"cpu": [
|
| 452 |
+
"x64"
|
| 453 |
+
],
|
| 454 |
+
"dev": true,
|
| 455 |
+
"license": "MIT",
|
| 456 |
+
"optional": true,
|
| 457 |
+
"os": [
|
| 458 |
+
"win32"
|
| 459 |
+
],
|
| 460 |
+
"engines": {
|
| 461 |
+
"node": ">=18"
|
| 462 |
+
}
|
| 463 |
+
},
|
| 464 |
+
"node_modules/@huggingface/tokenizers": {
|
| 465 |
+
"version": "0.2.0",
|
| 466 |
+
"resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.2.0.tgz",
|
| 467 |
+
"integrity": "sha512-LidMHe1FpcSYH4vcSjXooda34pC0M8m1gmDOv7SQi6Iv+ib1zX7J3sHBCC9/3FPCkn2k1b55FneVqMRxrr99pg==",
|
| 468 |
+
"license": "Apache-2.0"
|
| 469 |
+
},
|
| 470 |
+
"node_modules/@protobufjs/aspromise": {
|
| 471 |
+
"version": "1.1.2",
|
| 472 |
+
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
|
| 473 |
+
"integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
|
| 474 |
+
"license": "BSD-3-Clause"
|
| 475 |
+
},
|
| 476 |
+
"node_modules/@protobufjs/base64": {
|
| 477 |
+
"version": "1.1.2",
|
| 478 |
+
"resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
|
| 479 |
+
"integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
|
| 480 |
+
"license": "BSD-3-Clause"
|
| 481 |
+
},
|
| 482 |
+
"node_modules/@protobufjs/codegen": {
|
| 483 |
+
"version": "2.0.5",
|
| 484 |
+
"resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz",
|
| 485 |
+
"integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==",
|
| 486 |
+
"license": "BSD-3-Clause"
|
| 487 |
+
},
|
| 488 |
+
"node_modules/@protobufjs/eventemitter": {
|
| 489 |
+
"version": "1.1.1",
|
| 490 |
+
"resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz",
|
| 491 |
+
"integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==",
|
| 492 |
+
"license": "BSD-3-Clause"
|
| 493 |
+
},
|
| 494 |
+
"node_modules/@protobufjs/fetch": {
|
| 495 |
+
"version": "1.1.1",
|
| 496 |
+
"resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz",
|
| 497 |
+
"integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==",
|
| 498 |
+
"license": "BSD-3-Clause",
|
| 499 |
+
"dependencies": {
|
| 500 |
+
"@protobufjs/aspromise": "^1.1.1"
|
| 501 |
+
}
|
| 502 |
+
},
|
| 503 |
+
"node_modules/@protobufjs/float": {
|
| 504 |
+
"version": "1.0.2",
|
| 505 |
+
"resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
|
| 506 |
+
"integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
|
| 507 |
+
"license": "BSD-3-Clause"
|
| 508 |
+
},
|
| 509 |
+
"node_modules/@protobufjs/path": {
|
| 510 |
+
"version": "1.1.2",
|
| 511 |
+
"resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
|
| 512 |
+
"integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
|
| 513 |
+
"license": "BSD-3-Clause"
|
| 514 |
+
},
|
| 515 |
+
"node_modules/@protobufjs/pool": {
|
| 516 |
+
"version": "1.1.0",
|
| 517 |
+
"resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
|
| 518 |
+
"integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
|
| 519 |
+
"license": "BSD-3-Clause"
|
| 520 |
+
},
|
| 521 |
+
"node_modules/@protobufjs/utf8": {
|
| 522 |
+
"version": "1.1.2",
|
| 523 |
+
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz",
|
| 524 |
+
"integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==",
|
| 525 |
+
"license": "BSD-3-Clause"
|
| 526 |
+
},
|
| 527 |
+
"node_modules/@types/node": {
|
| 528 |
+
"version": "26.6.2",
|
| 529 |
+
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.2.tgz",
|
| 530 |
+
"integrity": "sha512-X1P21scMv4zGKLYqjdGjaKa7COa0RKVYYZZN/NfvLQ1JegxFhdhpZG/Lyn8AXx6CDUavKAd11v6BvfpkDByK8g==",
|
| 531 |
+
"license": "MIT",
|
| 532 |
+
"dependencies": {
|
| 533 |
+
"undici-types": "~8.9.0"
|
| 534 |
+
}
|
| 535 |
+
},
|
| 536 |
+
"node_modules/esbuild": {
|
| 537 |
+
"version": "0.28.2",
|
| 538 |
+
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz",
|
| 539 |
+
"integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==",
|
| 540 |
+
"dev": true,
|
| 541 |
+
"hasInstallScript": true,
|
| 542 |
+
"license": "MIT",
|
| 543 |
+
"bin": {
|
| 544 |
+
"esbuild": "bin/esbuild"
|
| 545 |
+
},
|
| 546 |
+
"engines": {
|
| 547 |
+
"node": ">=18"
|
| 548 |
+
},
|
| 549 |
+
"optionalDependencies": {
|
| 550 |
+
"@esbuild/aix-ppc64": "0.28.2",
|
| 551 |
+
"@esbuild/android-arm": "0.28.2",
|
| 552 |
+
"@esbuild/android-arm64": "0.28.2",
|
| 553 |
+
"@esbuild/android-x64": "0.28.2",
|
| 554 |
+
"@esbuild/darwin-arm64": "0.28.2",
|
| 555 |
+
"@esbuild/darwin-x64": "0.28.2",
|
| 556 |
+
"@esbuild/freebsd-arm64": "0.28.2",
|
| 557 |
+
"@esbuild/freebsd-x64": "0.28.2",
|
| 558 |
+
"@esbuild/linux-arm": "0.28.2",
|
| 559 |
+
"@esbuild/linux-arm64": "0.28.2",
|
| 560 |
+
"@esbuild/linux-ia32": "0.28.2",
|
| 561 |
+
"@esbuild/linux-loong64": "0.28.2",
|
| 562 |
+
"@esbuild/linux-mips64el": "0.28.2",
|
| 563 |
+
"@esbuild/linux-ppc64": "0.28.2",
|
| 564 |
+
"@esbuild/linux-riscv64": "0.28.2",
|
| 565 |
+
"@esbuild/linux-s390x": "0.28.2",
|
| 566 |
+
"@esbuild/linux-x64": "0.28.2",
|
| 567 |
+
"@esbuild/netbsd-arm64": "0.28.2",
|
| 568 |
+
"@esbuild/netbsd-x64": "0.28.2",
|
| 569 |
+
"@esbuild/openbsd-arm64": "0.28.2",
|
| 570 |
+
"@esbuild/openbsd-x64": "0.28.2",
|
| 571 |
+
"@esbuild/openharmony-arm64": "0.28.2",
|
| 572 |
+
"@esbuild/sunos-x64": "0.28.2",
|
| 573 |
+
"@esbuild/win32-arm64": "0.28.2",
|
| 574 |
+
"@esbuild/win32-ia32": "0.28.2",
|
| 575 |
+
"@esbuild/win32-x64": "0.28.2"
|
| 576 |
+
}
|
| 577 |
+
},
|
| 578 |
+
"node_modules/flatbuffers": {
|
| 579 |
+
"version": "25.9.23",
|
| 580 |
+
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz",
|
| 581 |
+
"integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==",
|
| 582 |
+
"license": "Apache-2.0"
|
| 583 |
+
},
|
| 584 |
+
"node_modules/guid-typescript": {
|
| 585 |
+
"version": "1.0.9",
|
| 586 |
+
"resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",
|
| 587 |
+
"integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==",
|
| 588 |
+
"license": "ISC"
|
| 589 |
+
},
|
| 590 |
+
"node_modules/long": {
|
| 591 |
+
"version": "5.3.2",
|
| 592 |
+
"resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
|
| 593 |
+
"integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
|
| 594 |
+
"license": "Apache-2.0"
|
| 595 |
+
},
|
| 596 |
+
"node_modules/onnxruntime-common": {
|
| 597 |
+
"version": "1.30.0",
|
| 598 |
+
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.30.0.tgz",
|
| 599 |
+
"integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==",
|
| 600 |
+
"license": "MIT"
|
| 601 |
+
},
|
| 602 |
+
"node_modules/onnxruntime-web": {
|
| 603 |
+
"version": "1.30.0",
|
| 604 |
+
"resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.30.0.tgz",
|
| 605 |
+
"integrity": "sha512-q0y+JrrtukXSzsBWEMccVfqX25LRmosXHF+CaRJmg8pZClzcV7svNc4rKY3jL02Vb7QmRMDs1SigqR4CXAfKYQ==",
|
| 606 |
+
"license": "MIT",
|
| 607 |
+
"dependencies": {
|
| 608 |
+
"flatbuffers": "^25.1.24",
|
| 609 |
+
"guid-typescript": "^1.0.9",
|
| 610 |
+
"long": "^5.2.3",
|
| 611 |
+
"onnxruntime-common": "1.30.0",
|
| 612 |
+
"platform": "^1.3.6",
|
| 613 |
+
"protobufjs": "^7.2.4"
|
| 614 |
+
}
|
| 615 |
+
},
|
| 616 |
+
"node_modules/platform": {
|
| 617 |
+
"version": "1.3.6",
|
| 618 |
+
"resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
|
| 619 |
+
"integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
|
| 620 |
+
"license": "MIT"
|
| 621 |
+
},
|
| 622 |
+
"node_modules/protobufjs": {
|
| 623 |
+
"version": "7.6.6",
|
| 624 |
+
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz",
|
| 625 |
+
"integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==",
|
| 626 |
+
"hasInstallScript": true,
|
| 627 |
+
"license": "BSD-3-Clause",
|
| 628 |
+
"dependencies": {
|
| 629 |
+
"@protobufjs/aspromise": "^1.1.2",
|
| 630 |
+
"@protobufjs/base64": "^1.1.2",
|
| 631 |
+
"@protobufjs/codegen": "^2.0.5",
|
| 632 |
+
"@protobufjs/eventemitter": "^1.1.1",
|
| 633 |
+
"@protobufjs/fetch": "^1.1.1",
|
| 634 |
+
"@protobufjs/float": "^1.0.2",
|
| 635 |
+
"@protobufjs/path": "^1.1.2",
|
| 636 |
+
"@protobufjs/pool": "^1.1.0",
|
| 637 |
+
"@protobufjs/utf8": "^1.1.1",
|
| 638 |
+
"@types/node": ">=13.7.0",
|
| 639 |
+
"long": "^5.3.2"
|
| 640 |
+
},
|
| 641 |
+
"engines": {
|
| 642 |
+
"node": ">=12.0.0"
|
| 643 |
+
}
|
| 644 |
+
},
|
| 645 |
+
"node_modules/undici-types": {
|
| 646 |
+
"version": "8.9.0",
|
| 647 |
+
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
|
| 648 |
+
"integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==",
|
| 649 |
+
"license": "MIT"
|
| 650 |
+
}
|
| 651 |
+
}
|
| 652 |
+
}
|
sdk/typescript/package.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "@vons/decision-sdk",
|
| 3 |
+
"version": "0.1.0",
|
| 4 |
+
"description": "Contract and host-boundary SDK for Vons decision models",
|
| 5 |
+
"type": "module",
|
| 6 |
+
"dependencies": {
|
| 7 |
+
"@huggingface/tokenizers": "0.2.0",
|
| 8 |
+
"onnxruntime-web": "1.30.0"
|
| 9 |
+
},
|
| 10 |
+
"exports": {
|
| 11 |
+
".": "./src/index.ts",
|
| 12 |
+
"./onnx": "./src/onnx.ts"
|
| 13 |
+
},
|
| 14 |
+
"scripts": {
|
| 15 |
+
"test": "node --experimental-strip-types --test test/index.test.mjs test/onnx.test.mjs test/parity.test.mjs",
|
| 16 |
+
"build:browser": "esbuild demo/main.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/main.js",
|
| 17 |
+
"build:benchmark": "esbuild demo/benchmark.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/benchmark.js",
|
| 18 |
+
"build:parity": "esbuild demo/parity.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/parity.js"
|
| 19 |
+
},
|
| 20 |
+
"engines": {
|
| 21 |
+
"node": ">=22"
|
| 22 |
+
},
|
| 23 |
+
"devDependencies": {
|
| 24 |
+
"esbuild": "0.28.2"
|
| 25 |
+
},
|
| 26 |
+
"private": true,
|
| 27 |
+
"files": [
|
| 28 |
+
"src/",
|
| 29 |
+
"README.md",
|
| 30 |
+
"LICENSE"
|
| 31 |
+
],
|
| 32 |
+
"license": "SEE LICENSE IN LICENSE"
|
| 33 |
+
}
|
sdk/typescript/src/index.ts
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
export type QuestionType = "choice" | "boolean" | "score";
|
| 2 |
+
export type Backend = "direct" | "diffusion";
|
| 3 |
+
export type ResponseStatus = "ok" | "abstain";
|
| 4 |
+
|
| 5 |
+
const validQuestionTypes = new Set<QuestionType>(["choice", "boolean", "score"]);
|
| 6 |
+
|
| 7 |
+
export interface Question {
|
| 8 |
+
id: string;
|
| 9 |
+
type: QuestionType;
|
| 10 |
+
prompt: string;
|
| 11 |
+
options: string[];
|
| 12 |
+
rubric?: string[];
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
export interface DecisionRequest {
|
| 16 |
+
state: string | Record<string, unknown>;
|
| 17 |
+
questions: Question[];
|
| 18 |
+
backend?: Backend;
|
| 19 |
+
seed?: number;
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
export interface OptionProbability {
|
| 23 |
+
option: string;
|
| 24 |
+
probability: number;
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
export interface QuestionAnswer {
|
| 28 |
+
question_id: string;
|
| 29 |
+
choice: string | null;
|
| 30 |
+
probabilities: OptionProbability[];
|
| 31 |
+
confidence: number;
|
| 32 |
+
status: ResponseStatus;
|
| 33 |
+
abstain_reason?: string | null;
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
export interface DecisionResponse {
|
| 37 |
+
answers: QuestionAnswer[];
|
| 38 |
+
backend: Backend;
|
| 39 |
+
model_id: string;
|
| 40 |
+
metadata?: Record<string, unknown>;
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
export type ToolRisk = "low" | "medium" | "high" | "critical";
|
| 44 |
+
|
| 45 |
+
export interface ToolCall {
|
| 46 |
+
name: string;
|
| 47 |
+
arguments: Record<string, unknown>;
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
export interface KASIProposal {
|
| 51 |
+
calls?: ToolCall[];
|
| 52 |
+
confidence: number;
|
| 53 |
+
risk?: string | null;
|
| 54 |
+
response?: string | null;
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
export type KASIAction = "call" | "clarify" | "confirm" | "refuse" | "respond";
|
| 58 |
+
|
| 59 |
+
export interface KASIActionDecision {
|
| 60 |
+
action: KASIAction;
|
| 61 |
+
calls: ToolCall[];
|
| 62 |
+
prompt: string | null;
|
| 63 |
+
reason: string;
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
export interface DecisionBackend {
|
| 67 |
+
readonly backend: Backend;
|
| 68 |
+
decide(request: DecisionRequest): Promise<DecisionResponse>;
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
const toolNamePattern = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/;
|
| 72 |
+
const validRisks = new Set<ToolRisk>(["low", "medium", "high", "critical"]);
|
| 73 |
+
const riskRank: Record<ToolRisk, number> = { low: 0, medium: 1, high: 2, critical: 3 };
|
| 74 |
+
|
| 75 |
+
function assertFiniteUnit(value: number, name: string): void {
|
| 76 |
+
if (!Number.isFinite(value) || value < 0 || value > 1) {
|
| 77 |
+
throw new TypeError(`${name} must be finite and in [0, 1]`);
|
| 78 |
+
}
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
function assertToolName(name: string): void {
|
| 82 |
+
if (typeof name !== "string" || !toolNamePattern.test(name)) {
|
| 83 |
+
throw new TypeError("tool name must match [A-Za-z0-9][A-Za-z0-9_.:-]{0,127}");
|
| 84 |
+
}
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
function normalizeRisk(value: string | null | undefined): ToolRisk | null {
|
| 88 |
+
return value && validRisks.has(value.trim().toLowerCase() as ToolRisk)
|
| 89 |
+
? (value.trim().toLowerCase() as ToolRisk)
|
| 90 |
+
: null;
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
function stableJson(value: unknown): string {
|
| 94 |
+
if (value === null || typeof value !== "object") {
|
| 95 |
+
const primitive = JSON.stringify(value);
|
| 96 |
+
if (primitive === undefined) throw new TypeError("arguments must be JSON-compatible");
|
| 97 |
+
return primitive;
|
| 98 |
+
}
|
| 99 |
+
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
| 100 |
+
const entries = Object.entries(value as Record<string, unknown>).sort(([a], [b]) => a.localeCompare(b));
|
| 101 |
+
return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(",")}}`;
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
export function callFingerprint(call: ToolCall): string {
|
| 105 |
+
assertToolName(call.name);
|
| 106 |
+
// SHA-256 is supplied by the host in production. The portable SDK key is a
|
| 107 |
+
// canonical consent key; callers may hash it before persistence.
|
| 108 |
+
return stableJson({ name: call.name, arguments: call.arguments });
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
export function validateRequest(request: DecisionRequest, maxQuestions = 8, maxTokens = 512): void {
|
| 112 |
+
if (!request || typeof request !== "object") throw new TypeError("request must be an object");
|
| 113 |
+
if (!Number.isInteger(maxQuestions) || maxQuestions < 1) throw new TypeError("maxQuestions must be a positive integer");
|
| 114 |
+
if (!Number.isInteger(maxTokens) || maxTokens < 1) throw new TypeError("maxTokens must be a positive integer");
|
| 115 |
+
if (typeof request.state !== "string" && (!request.state || typeof request.state !== "object" || Array.isArray(request.state))) {
|
| 116 |
+
throw new TypeError("state must be a string or object");
|
| 117 |
+
}
|
| 118 |
+
if (typeof request.state === "string" && request.state.trim().length === 0) {
|
| 119 |
+
throw new TypeError("state must not be empty");
|
| 120 |
+
}
|
| 121 |
+
if (!Array.isArray(request.questions) || request.questions.length < 1 || request.questions.length > maxQuestions) {
|
| 122 |
+
throw new TypeError(`questions must contain 1..${maxQuestions} items`);
|
| 123 |
+
}
|
| 124 |
+
const ids = new Set<string>();
|
| 125 |
+
for (const question of request.questions) {
|
| 126 |
+
if (!question || typeof question !== "object") throw new TypeError("questions must contain objects");
|
| 127 |
+
if (typeof question.id !== "string" || !question.id || ids.has(question.id)) throw new TypeError("question ids must be unique and non-empty");
|
| 128 |
+
ids.add(question.id);
|
| 129 |
+
if (!validQuestionTypes.has(question.type)) throw new TypeError(`question ${question.id} has an invalid type`);
|
| 130 |
+
if (typeof question.prompt !== "string" || !question.prompt.trim()) throw new TypeError(`question ${question.id} has an empty prompt`);
|
| 131 |
+
if (!Array.isArray(question.options) || question.options.some((option) => typeof option !== "string" || !option.trim())) {
|
| 132 |
+
throw new TypeError(`question ${question.id} options must be strings`);
|
| 133 |
+
}
|
| 134 |
+
if (question.type === "boolean") {
|
| 135 |
+
if (question.options.length > 0 && JSON.stringify(question.options) !== JSON.stringify(["true", "false"])) {
|
| 136 |
+
throw new TypeError("boolean questions must use true/false options");
|
| 137 |
+
}
|
| 138 |
+
} else if (!Array.isArray(question.options) || question.options.length < 2 || question.options.length > 32) {
|
| 139 |
+
throw new TypeError(`question ${question.id} must contain 2..32 options`);
|
| 140 |
+
}
|
| 141 |
+
if (new Set(question.options).size !== question.options.length) throw new TypeError("question options must be unique");
|
| 142 |
+
if (question.rubric !== undefined && !Array.isArray(question.rubric)) {
|
| 143 |
+
throw new TypeError("question rubric must be a list");
|
| 144 |
+
}
|
| 145 |
+
if (question.type === "score" && (!question.rubric || question.rubric.length < 2 || question.rubric.length > 10)) {
|
| 146 |
+
throw new TypeError("score questions must contain a 2..10 item rubric");
|
| 147 |
+
}
|
| 148 |
+
if (question.type === "score" && question.rubric?.some((item) => typeof item !== "string")) {
|
| 149 |
+
throw new TypeError("score question rubric items must be strings");
|
| 150 |
+
}
|
| 151 |
+
if (question.type !== "score" && question.rubric?.length) {
|
| 152 |
+
throw new TypeError(`only score question ${question.id} may provide rubric`);
|
| 153 |
+
}
|
| 154 |
+
}
|
| 155 |
+
// Tokenizer-specific budget checks belong to the runtime adapter. A character
|
| 156 |
+
// estimate here would reject valid WordPiece inputs and cannot account for
|
| 157 |
+
// the aggregate state/question/all-candidate contract. Keep maxTokens in the
|
| 158 |
+
// signature for API compatibility and let adapters enforce their exact limit.
|
| 159 |
+
void maxTokens;
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
export function validateResponse(response: DecisionResponse): void {
|
| 163 |
+
if (!response || typeof response !== "object") throw new TypeError("response must be an object");
|
| 164 |
+
if (typeof response.model_id !== "string" || !response.model_id.trim()) throw new TypeError("model_id is required");
|
| 165 |
+
if (response.backend !== "direct" && response.backend !== "diffusion") throw new TypeError("invalid backend");
|
| 166 |
+
if (!Array.isArray(response.answers)) throw new TypeError("answers must be a list");
|
| 167 |
+
if (response.metadata !== undefined && (!response.metadata || typeof response.metadata !== "object" || Array.isArray(response.metadata))) {
|
| 168 |
+
throw new TypeError("metadata must be an object");
|
| 169 |
+
}
|
| 170 |
+
const ids = new Set<string>();
|
| 171 |
+
for (const answer of response.answers) {
|
| 172 |
+
if (!answer || typeof answer !== "object") throw new TypeError("answers must contain objects");
|
| 173 |
+
if (typeof answer.question_id !== "string" || !answer.question_id || ids.has(answer.question_id)) throw new TypeError("answer question ids must be unique and non-empty");
|
| 174 |
+
ids.add(answer.question_id);
|
| 175 |
+
if (!Array.isArray(answer.probabilities)) throw new TypeError("probabilities must be a list");
|
| 176 |
+
assertFiniteUnit(answer.confidence, "confidence");
|
| 177 |
+
const options = new Set<string>();
|
| 178 |
+
let total = 0;
|
| 179 |
+
for (const item of answer.probabilities) {
|
| 180 |
+
if (!item || typeof item !== "object" || typeof item.option !== "string" || !item.option) {
|
| 181 |
+
throw new TypeError("probability options must be non-empty strings");
|
| 182 |
+
}
|
| 183 |
+
if (options.has(item.option)) throw new TypeError("probability options must be unique");
|
| 184 |
+
options.add(item.option);
|
| 185 |
+
assertFiniteUnit(item.probability, "probability");
|
| 186 |
+
total += item.probability;
|
| 187 |
+
}
|
| 188 |
+
if (answer.probabilities.length > 0 && Math.abs(total - 1) > 1e-6) throw new TypeError("probabilities must sum to 1");
|
| 189 |
+
if (answer.status !== "ok" && answer.status !== "abstain") throw new TypeError("invalid response status");
|
| 190 |
+
if (answer.choice !== null && typeof answer.choice !== "string") throw new TypeError("choice must be a string or null");
|
| 191 |
+
if (answer.abstain_reason !== undefined && answer.abstain_reason !== null && typeof answer.abstain_reason !== "string") {
|
| 192 |
+
throw new TypeError("abstain_reason must be a string or null");
|
| 193 |
+
}
|
| 194 |
+
if (answer.status === "ok" && !answer.choice) throw new TypeError("ok answers require choice");
|
| 195 |
+
if (answer.status === "abstain" && (answer.choice !== null || !answer.abstain_reason?.trim())) {
|
| 196 |
+
throw new TypeError("abstain answers require a reason and no choice");
|
| 197 |
+
}
|
| 198 |
+
if (answer.status === "ok" && answer.probabilities.length > 0 && !options.has(answer.choice!)) {
|
| 199 |
+
throw new TypeError("choice must be present in probabilities");
|
| 200 |
+
}
|
| 201 |
+
}
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
export class KASIAdapter {
|
| 205 |
+
private readonly policy: ReadonlyMap<string, ToolRisk>;
|
| 206 |
+
private readonly threshold: number;
|
| 207 |
+
|
| 208 |
+
constructor(policy: Record<string, ToolRisk>, threshold = 0.55) {
|
| 209 |
+
if (!Number.isFinite(threshold) || threshold <= 0 || threshold > 1) throw new TypeError("invalid threshold");
|
| 210 |
+
this.threshold = threshold;
|
| 211 |
+
const entries: [string, ToolRisk][] = [];
|
| 212 |
+
for (const [name, risk] of Object.entries(policy)) {
|
| 213 |
+
assertToolName(name);
|
| 214 |
+
if (!validRisks.has(risk)) throw new TypeError(`invalid risk for ${name}`);
|
| 215 |
+
entries.push([name, risk]);
|
| 216 |
+
}
|
| 217 |
+
this.policy = new Map(entries);
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
decide(proposal: KASIProposal, confirmedCalls: ReadonlySet<string> = new Set()): KASIActionDecision {
|
| 221 |
+
assertFiniteUnit(proposal.confidence, "confidence");
|
| 222 |
+
const calls = proposal.calls ?? [];
|
| 223 |
+
if (calls.length > 0 && proposal.response) throw new TypeError("proposal cannot contain both calls and response");
|
| 224 |
+
if (proposal.response && !(proposal.calls?.length)) {
|
| 225 |
+
if (proposal.confidence >= this.threshold) return { action: "respond", calls: [], prompt: proposal.response, reason: "direct_response" };
|
| 226 |
+
}
|
| 227 |
+
if (proposal.confidence < this.threshold) return { action: "clarify", calls, prompt: "Please provide the missing information before acting.", reason: "low_confidence" };
|
| 228 |
+
if (calls.length === 0) return { action: "refuse", calls, prompt: "No safe candidate action is available.", reason: "no_proposed_call" };
|
| 229 |
+
const advisory = normalizeRisk(proposal.risk);
|
| 230 |
+
if (proposal.risk !== null && proposal.risk !== undefined && !advisory) {
|
| 231 |
+
return { action: "confirm", calls, prompt: "Confirm action with unknown model risk.", reason: "unknown_model_risk" };
|
| 232 |
+
}
|
| 233 |
+
for (const call of calls) {
|
| 234 |
+
assertToolName(call.name);
|
| 235 |
+
const hostRisk = this.policy.get(call.name);
|
| 236 |
+
if (!hostRisk) return { action: "refuse", calls: [], prompt: "Tool is not registered by the host.", reason: "unregistered_tool" };
|
| 237 |
+
const needsConsent = !advisory || Math.max(riskRank[hostRisk], riskRank[advisory]) >= riskRank.high;
|
| 238 |
+
if (needsConsent && !confirmedCalls.has(callFingerprint(call))) {
|
| 239 |
+
return { action: "confirm", calls, prompt: `Confirm action: ${call.name}(${stableJson(call.arguments)}).`, reason: "unconfirmed_or_unknown_risk" };
|
| 240 |
+
}
|
| 241 |
+
}
|
| 242 |
+
return { action: "call", calls, prompt: null, reason: "approved_by_host_policy" };
|
| 243 |
+
}
|
| 244 |
+
}
|
| 245 |
+
|
| 246 |
+
export function selectRuntime(preferred: "wasm" | "webgpu" = "wasm"): "wasm" | "webgpu" {
|
| 247 |
+
if (preferred === "webgpu") {
|
| 248 |
+
const scope = globalThis as { navigator?: { gpu?: unknown }; gpu?: unknown };
|
| 249 |
+
const gpu = scope.navigator?.gpu ?? scope.gpu;
|
| 250 |
+
if (!gpu) throw new Error("WebGPU is unavailable; choose wasm explicitly");
|
| 251 |
+
return "webgpu";
|
| 252 |
+
}
|
| 253 |
+
if (typeof WebAssembly === "undefined") throw new Error("WebAssembly is unavailable");
|
| 254 |
+
return "wasm";
|
| 255 |
+
}
|
sdk/typescript/src/onnx.ts
ADDED
|
@@ -0,0 +1,606 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { Tokenizer } from "@huggingface/tokenizers";
|
| 2 |
+
|
| 3 |
+
import {
|
| 4 |
+
type Backend,
|
| 5 |
+
type DecisionBackend,
|
| 6 |
+
type DecisionRequest,
|
| 7 |
+
type DecisionResponse,
|
| 8 |
+
type Question,
|
| 9 |
+
type QuestionAnswer,
|
| 10 |
+
validateRequest,
|
| 11 |
+
validateResponse,
|
| 12 |
+
} from "./index.ts";
|
| 13 |
+
|
| 14 |
+
export type OnnxExecutionProvider = "wasm" | "webgpu";
|
| 15 |
+
|
| 16 |
+
export interface BundleFileRecord {
|
| 17 |
+
path: string;
|
| 18 |
+
role: string;
|
| 19 |
+
bytes: number;
|
| 20 |
+
sha256: string;
|
| 21 |
+
runtime_loaded: boolean;
|
| 22 |
+
model_asset: boolean;
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
export interface BundleManifest {
|
| 26 |
+
schema_version: "vons.bundle.manifest/v1";
|
| 27 |
+
manifest: { path: string; sha256: string | null; self_hash_excluded: boolean };
|
| 28 |
+
metadata: {
|
| 29 |
+
backend: Backend;
|
| 30 |
+
model_id?: string;
|
| 31 |
+
option_count?: number;
|
| 32 |
+
sequence_length?: number;
|
| 33 |
+
input_names?: string[];
|
| 34 |
+
output_names?: string[];
|
| 35 |
+
graph?: {
|
| 36 |
+
external_data_locations?: string[];
|
| 37 |
+
inputs?: Array<{ name: string; dtype: string; shape: Array<number | string | null> | null }>;
|
| 38 |
+
outputs?: Array<{ name: string; dtype: string; shape: Array<number | string | null> | null }>;
|
| 39 |
+
};
|
| 40 |
+
[key: string]: unknown;
|
| 41 |
+
};
|
| 42 |
+
files: BundleFileRecord[];
|
| 43 |
+
summary: {
|
| 44 |
+
model_asset_bytes: number;
|
| 45 |
+
complete_download_bytes: number;
|
| 46 |
+
[key: string]: unknown;
|
| 47 |
+
};
|
| 48 |
+
[key: string]: unknown;
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
export interface TokenizedCandidate {
|
| 52 |
+
inputIds: number[];
|
| 53 |
+
attentionMask: number[];
|
| 54 |
+
tokenTypeIds: number[];
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
export interface OnnxWebBackendOptions {
|
| 58 |
+
/** Use a URL when the manifest is served beside the browser bundle. */
|
| 59 |
+
manifestUrl?: string | URL;
|
| 60 |
+
/** Supply a previously fetched manifest when the caller controls loading. */
|
| 61 |
+
manifest?: BundleManifest;
|
| 62 |
+
/** Pin the exact manifest bytes expected by a release or measurement run. */
|
| 63 |
+
expectedManifestSha256?: string;
|
| 64 |
+
/** Required with an in-memory manifest unless the browser has a useful base URI. */
|
| 65 |
+
baseUrl?: string | URL;
|
| 66 |
+
provider?: OnnxExecutionProvider;
|
| 67 |
+
wasmPaths?: string | Record<string, string>;
|
| 68 |
+
wasmNumThreads?: number;
|
| 69 |
+
abstainThreshold?: number;
|
| 70 |
+
answerabilityThreshold?: number;
|
| 71 |
+
/** Override Direct's live candidate count for padding-cost experiments. */
|
| 72 |
+
directOptionSlots?: number;
|
| 73 |
+
/** Freeze the exact noise array when a cross-runtime comparison needs it. */
|
| 74 |
+
initialNoise?: (length: number, seed: number | undefined, questionIndex: number) => Float32Array;
|
| 75 |
+
/** Optional instrumentation hook; it is not part of the decision response contract. */
|
| 76 |
+
onTiming?: (sample: OnnxTimingSample) => void;
|
| 77 |
+
/** Optional local parity trace; disabled by default and not part of the response contract. */
|
| 78 |
+
onTrace?: (sample: OnnxTraceSample) => void;
|
| 79 |
+
fetch?: typeof fetch;
|
| 80 |
+
signal?: AbortSignal;
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
export interface OnnxRuntimeInfo {
|
| 84 |
+
provider: OnnxExecutionProvider;
|
| 85 |
+
ortVersion: string | null;
|
| 86 |
+
wasmNumThreads: number | null;
|
| 87 |
+
webgpuAdapterRequested: boolean;
|
| 88 |
+
webgpuDeviceRequested: boolean;
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
export interface OnnxTimingSample {
|
| 92 |
+
questionId: string;
|
| 93 |
+
live_candidates: number;
|
| 94 |
+
allocated_candidates: number;
|
| 95 |
+
live_tokens: number;
|
| 96 |
+
sequence_length: number;
|
| 97 |
+
tokenizeMs: number;
|
| 98 |
+
prepareFeedMs: number;
|
| 99 |
+
inferenceAndReadbackMs: number;
|
| 100 |
+
postprocessMs: number;
|
| 101 |
+
totalRequestMs: number;
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
export interface OnnxTraceSample {
|
| 105 |
+
questionId: string;
|
| 106 |
+
inputIds: number[];
|
| 107 |
+
attentionMask: number[];
|
| 108 |
+
tokenTypeIds: number[];
|
| 109 |
+
optionMask: number[];
|
| 110 |
+
initialNoise: number[] | null;
|
| 111 |
+
rawScores: number[];
|
| 112 |
+
answerabilityLogit: number;
|
| 113 |
+
probabilities: number[];
|
| 114 |
+
answer: QuestionAnswer;
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
interface OrtTensor {
|
| 118 |
+
data: ArrayLike<number>;
|
| 119 |
+
dims: readonly number[];
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
interface OrtSession {
|
| 123 |
+
run(feeds: Record<string, unknown>): Promise<Record<string, OrtTensor>>;
|
| 124 |
+
release(): Promise<void>;
|
| 125 |
+
inputNames: readonly string[];
|
| 126 |
+
outputNames: readonly string[];
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
interface OrtModule {
|
| 130 |
+
Tensor: new (type: string, data: ArrayLike<number> | ArrayLike<bigint>, dims: readonly number[]) => unknown;
|
| 131 |
+
InferenceSession: { create(model: ArrayBufferLike, options?: Record<string, unknown>): Promise<OrtSession> };
|
| 132 |
+
env: {
|
| 133 |
+
wasm: { wasmPaths?: string | Record<string, string>; numThreads?: number };
|
| 134 |
+
versions?: { common?: string; web?: string };
|
| 135 |
+
};
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
interface GpuLike {
|
| 139 |
+
requestAdapter(): Promise<{ requestDevice(): Promise<unknown> } | null>;
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
interface LoadedAssets {
|
| 143 |
+
manifest: BundleManifest;
|
| 144 |
+
manifestHash: string;
|
| 145 |
+
model: Uint8Array;
|
| 146 |
+
tokenizer: Tokenizer;
|
| 147 |
+
externalData: Array<{ path: string; data: Uint8Array }>;
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
const EXPECTED_SCHEMA = "vons.bundle.manifest/v1";
|
| 151 |
+
const REQUIRED_DIRECT_INPUTS = ["input_ids", "attention_mask", "token_type_ids", "option_mask"];
|
| 152 |
+
const REQUIRED_DIFFUSION_INPUTS = [...REQUIRED_DIRECT_INPUTS, "initial_noise"];
|
| 153 |
+
|
| 154 |
+
function assertFiniteUnit(value: number, name: string): void {
|
| 155 |
+
if (!Number.isFinite(value) || value < 0 || value > 1) throw new TypeError(`${name} must be finite and in [0, 1]`);
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
function stableJson(value: unknown): string {
|
| 159 |
+
if (value === null || typeof value !== "object") {
|
| 160 |
+
const primitive = JSON.stringify(value);
|
| 161 |
+
if (primitive === undefined) throw new TypeError("state must be JSON-compatible");
|
| 162 |
+
return primitive;
|
| 163 |
+
}
|
| 164 |
+
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
| 165 |
+
const entries = Object.entries(value as Record<string, unknown>).sort(([a], [b]) => a.localeCompare(b));
|
| 166 |
+
return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(",")}}`;
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
export function serializeState(state: DecisionRequest["state"]): string {
|
| 170 |
+
return typeof state === "string" ? state : stableJson(state);
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
export function candidateText(state: DecisionRequest["state"], question: Question, option: string): string {
|
| 174 |
+
return `${serializeState(state)}\nQuestion: ${question.prompt}\nCandidate: ${option}`;
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
export function candidateListText(state: DecisionRequest["state"], question: Question, options: readonly string[]): string {
|
| 178 |
+
return `${serializeState(state)}\nQuestion: ${question.prompt}\nCandidates:\n${options.join("\n")}`;
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
function assertManifest(manifest: BundleManifest): void {
|
| 182 |
+
if (!manifest || manifest.schema_version !== EXPECTED_SCHEMA) {
|
| 183 |
+
throw new TypeError(`unsupported bundle manifest schema: ${manifest?.schema_version ?? "missing"}`);
|
| 184 |
+
}
|
| 185 |
+
if (!manifest.metadata || (manifest.metadata.backend !== "direct" && manifest.metadata.backend !== "diffusion")) {
|
| 186 |
+
throw new TypeError("bundle manifest metadata.backend must be direct or diffusion");
|
| 187 |
+
}
|
| 188 |
+
if (!Array.isArray(manifest.files) || manifest.files.length === 0) throw new TypeError("bundle manifest has no files");
|
| 189 |
+
const paths = new Set<string>();
|
| 190 |
+
for (const item of manifest.files) {
|
| 191 |
+
if (!item || typeof item.path !== "string" || item.path.length === 0 || paths.has(item.path)) {
|
| 192 |
+
throw new TypeError(`bundle manifest contains an invalid or duplicate path: ${item?.path ?? "missing"}`);
|
| 193 |
+
}
|
| 194 |
+
if (item.path.startsWith("/") || item.path.includes("..") || /^[A-Za-z]:[\\/]/.test(item.path)) {
|
| 195 |
+
throw new TypeError(`bundle manifest path must be bundle-relative: ${item.path}`);
|
| 196 |
+
}
|
| 197 |
+
if (!Number.isSafeInteger(item.bytes) || item.bytes < 0 || !/^[0-9a-f]{64}$/.test(item.sha256)) {
|
| 198 |
+
throw new TypeError(`bundle manifest has invalid bytes or sha256: ${item.path}`);
|
| 199 |
+
}
|
| 200 |
+
paths.add(item.path);
|
| 201 |
+
}
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
function getBaseUrl(options: OnnxWebBackendOptions): URL {
|
| 205 |
+
if (options.baseUrl) return new URL(options.baseUrl.toString(), globalThis.location?.href ?? "http://localhost/");
|
| 206 |
+
if (options.manifestUrl) return new URL(options.manifestUrl.toString(), globalThis.location?.href ?? "http://localhost/");
|
| 207 |
+
if (typeof globalThis.location?.href === "string") return new URL(globalThis.location.href);
|
| 208 |
+
throw new TypeError("baseUrl or manifestUrl is required when running outside a browser");
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
function resolveAssetUrl(base: URL, path: string): URL {
|
| 212 |
+
if (
|
| 213 |
+
path.startsWith("/") ||
|
| 214 |
+
path.includes("\\") ||
|
| 215 |
+
path.includes("..") ||
|
| 216 |
+
/%(?:2e|2f|5c)/i.test(path) ||
|
| 217 |
+
/[?#]/.test(path) ||
|
| 218 |
+
/^[A-Za-z]:[\\/]/.test(path) ||
|
| 219 |
+
/^[A-Za-z][A-Za-z+.-]*:/.test(path)
|
| 220 |
+
) {
|
| 221 |
+
throw new TypeError(`bundle asset path must be relative: ${path}`);
|
| 222 |
+
}
|
| 223 |
+
const resolved = new URL(path, base);
|
| 224 |
+
const basePath = base.pathname.endsWith("/") ? base.pathname : `${base.pathname}/`;
|
| 225 |
+
if (resolved.origin !== base.origin || !resolved.pathname.startsWith(basePath)) {
|
| 226 |
+
throw new TypeError(`bundle asset path escapes its manifest directory: ${path}`);
|
| 227 |
+
}
|
| 228 |
+
return resolved;
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
async function sha256(bytes: Uint8Array): Promise<string> {
|
| 232 |
+
if (!globalThis.crypto?.subtle) throw new Error("Web Crypto SHA-256 is unavailable");
|
| 233 |
+
const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes as unknown as BufferSource);
|
| 234 |
+
return Array.from(new Uint8Array(digest), (value) => value.toString(16).padStart(2, "0")).join("");
|
| 235 |
+
}
|
| 236 |
+
|
| 237 |
+
async function fetchBytes(fetchImpl: typeof fetch, url: URL, signal?: AbortSignal): Promise<Uint8Array> {
|
| 238 |
+
const response = await fetchImpl(url, { signal });
|
| 239 |
+
if (!response.ok) throw new Error(`failed to fetch ${url}: HTTP ${response.status}`);
|
| 240 |
+
return new Uint8Array(await response.arrayBuffer());
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
async function loadManifest(options: OnnxWebBackendOptions, fetchImpl: typeof fetch): Promise<{ manifest: BundleManifest; manifestHash: string; base: URL }> {
|
| 244 |
+
if (options.expectedManifestSha256 !== undefined && !/^[0-9a-f]{64}$/.test(options.expectedManifestSha256)) {
|
| 245 |
+
throw new TypeError("expectedManifestSha256 must be a lowercase SHA-256 digest");
|
| 246 |
+
}
|
| 247 |
+
if (options.manifest) {
|
| 248 |
+
assertManifest(options.manifest);
|
| 249 |
+
const manifestBytes = new TextEncoder().encode(stableJson(options.manifest));
|
| 250 |
+
const manifestHash = await sha256(manifestBytes);
|
| 251 |
+
if (options.expectedManifestSha256 !== undefined && manifestHash !== options.expectedManifestSha256) {
|
| 252 |
+
throw new Error(`bundle manifest SHA-256 mismatch: ${manifestHash}`);
|
| 253 |
+
}
|
| 254 |
+
return { manifest: options.manifest, base: getBaseUrl(options), manifestHash };
|
| 255 |
+
}
|
| 256 |
+
if (!options.manifestUrl) throw new TypeError("manifest or manifestUrl is required");
|
| 257 |
+
const url = new URL(options.manifestUrl.toString(), globalThis.location?.href ?? "http://localhost/");
|
| 258 |
+
const bytes = await fetchBytes(fetchImpl, url, options.signal);
|
| 259 |
+
let manifest: BundleManifest;
|
| 260 |
+
try {
|
| 261 |
+
manifest = JSON.parse(new TextDecoder().decode(bytes)) as BundleManifest;
|
| 262 |
+
} catch (error) {
|
| 263 |
+
throw new TypeError(`bundle manifest is not valid JSON: ${String(error)}`);
|
| 264 |
+
}
|
| 265 |
+
assertManifest(manifest);
|
| 266 |
+
const manifestHash = await sha256(bytes);
|
| 267 |
+
if (options.expectedManifestSha256 !== undefined && manifestHash !== options.expectedManifestSha256) {
|
| 268 |
+
throw new Error(`bundle manifest SHA-256 mismatch: ${manifestHash}`);
|
| 269 |
+
}
|
| 270 |
+
return { manifest, base: new URL(".", url), manifestHash };
|
| 271 |
+
}
|
| 272 |
+
|
| 273 |
+
async function loadAssets(options: OnnxWebBackendOptions): Promise<LoadedAssets> {
|
| 274 |
+
const fetchImpl = options.fetch ?? globalThis.fetch?.bind(globalThis);
|
| 275 |
+
if (!fetchImpl) throw new Error("fetch is unavailable; provide options.fetch");
|
| 276 |
+
const { manifest, manifestHash, base } = await loadManifest(options, fetchImpl);
|
| 277 |
+
const cache = new Map<string, Uint8Array>();
|
| 278 |
+
const getFile = async (record: BundleFileRecord): Promise<Uint8Array> => {
|
| 279 |
+
const cached = cache.get(record.path);
|
| 280 |
+
if (cached) return cached;
|
| 281 |
+
const bytes = await fetchBytes(fetchImpl, resolveAssetUrl(base, record.path), options.signal);
|
| 282 |
+
if (bytes.byteLength !== record.bytes) throw new Error(`bundle byte count mismatch: ${record.path}`);
|
| 283 |
+
const actual = await sha256(bytes);
|
| 284 |
+
if (actual !== record.sha256) throw new Error(`bundle SHA-256 mismatch: ${record.path}`);
|
| 285 |
+
cache.set(record.path, bytes);
|
| 286 |
+
return bytes;
|
| 287 |
+
};
|
| 288 |
+
const graph = manifest.files.find((item) => item.role === "model_graph");
|
| 289 |
+
if (!graph) throw new Error("bundle manifest has no model graph");
|
| 290 |
+
const tokenizerJson = manifest.files.find((item) => item.role === "tokenizer" && item.path.endsWith("tokenizer.json"));
|
| 291 |
+
const tokenizerConfig = manifest.files.find((item) => item.role === "tokenizer" && item.path.endsWith("tokenizer_config.json"));
|
| 292 |
+
if (!tokenizerJson || !tokenizerConfig) throw new Error("bundle manifest must include tokenizer.json and tokenizer_config.json");
|
| 293 |
+
const model = await getFile(graph);
|
| 294 |
+
const tokenizer = new Tokenizer(
|
| 295 |
+
JSON.parse(new TextDecoder().decode(await getFile(tokenizerJson))) as object,
|
| 296 |
+
JSON.parse(new TextDecoder().decode(await getFile(tokenizerConfig))) as object,
|
| 297 |
+
);
|
| 298 |
+
const locations = manifest.metadata.graph?.external_data_locations ?? [];
|
| 299 |
+
const externalData: Array<{ path: string; data: Uint8Array }> = [];
|
| 300 |
+
for (const location of locations) {
|
| 301 |
+
const record = manifest.files.find((item) => item.role === "model_external_data" && item.path === location);
|
| 302 |
+
if (!record) throw new Error(`external-data location is missing from the manifest: ${location}`);
|
| 303 |
+
externalData.push({ path: location, data: await getFile(record) });
|
| 304 |
+
}
|
| 305 |
+
return { manifest, manifestHash, model, tokenizer, externalData };
|
| 306 |
+
}
|
| 307 |
+
|
| 308 |
+
function encodeCandidate(tokenizer: Tokenizer, text: string, sequenceLength: number): TokenizedCandidate {
|
| 309 |
+
const encoded = tokenizer.encode(text, { return_token_type_ids: true });
|
| 310 |
+
const inputIds = [...encoded.ids];
|
| 311 |
+
const attentionMask = [...encoded.attention_mask];
|
| 312 |
+
const tokenTypeIds = [...encoded.token_type_ids];
|
| 313 |
+
if (inputIds.length !== attentionMask.length || inputIds.length !== tokenTypeIds.length) {
|
| 314 |
+
throw new Error("tokenizer returned arrays with different lengths");
|
| 315 |
+
}
|
| 316 |
+
if (inputIds.length > sequenceLength) {
|
| 317 |
+
throw new RangeError(`tokenized candidate has ${inputIds.length} tokens; the bundle limit is ${sequenceLength}`);
|
| 318 |
+
}
|
| 319 |
+
if (inputIds.some((value) => !Number.isSafeInteger(value) || value < 0)) throw new Error("tokenizer returned an invalid token id");
|
| 320 |
+
return { inputIds, attentionMask, tokenTypeIds };
|
| 321 |
+
}
|
| 322 |
+
|
| 323 |
+
export function tokenizeCandidates(
|
| 324 |
+
tokenizer: Tokenizer,
|
| 325 |
+
state: DecisionRequest["state"],
|
| 326 |
+
question: Question,
|
| 327 |
+
options: readonly string[],
|
| 328 |
+
sequenceLength: number,
|
| 329 |
+
): TokenizedCandidate[] {
|
| 330 |
+
return options.map((option) => encodeCandidate(tokenizer, candidateText(state, question, option), sequenceLength));
|
| 331 |
+
}
|
| 332 |
+
|
| 333 |
+
function padCandidates(candidates: readonly TokenizedCandidate[], slots: number, sequenceLength: number, paddingId: number): {
|
| 334 |
+
inputIds: BigInt64Array;
|
| 335 |
+
attentionMask: BigInt64Array;
|
| 336 |
+
tokenTypeIds: BigInt64Array;
|
| 337 |
+
optionMask: Uint8Array;
|
| 338 |
+
} {
|
| 339 |
+
if (!Number.isInteger(slots) || slots < candidates.length || slots < 1) throw new RangeError("invalid candidate slot count");
|
| 340 |
+
const inputIds = new BigInt64Array(slots * sequenceLength);
|
| 341 |
+
const attentionMask = new BigInt64Array(slots * sequenceLength);
|
| 342 |
+
const tokenTypeIds = new BigInt64Array(slots * sequenceLength);
|
| 343 |
+
const optionMask = new Uint8Array(slots);
|
| 344 |
+
inputIds.fill(BigInt(paddingId));
|
| 345 |
+
for (let optionIndex = 0; optionIndex < candidates.length; optionIndex += 1) {
|
| 346 |
+
const candidate = candidates[optionIndex];
|
| 347 |
+
optionMask[optionIndex] = 1;
|
| 348 |
+
const offset = optionIndex * sequenceLength;
|
| 349 |
+
for (let tokenIndex = 0; tokenIndex < candidate.inputIds.length; tokenIndex += 1) {
|
| 350 |
+
inputIds[offset + tokenIndex] = BigInt(candidate.inputIds[tokenIndex]);
|
| 351 |
+
attentionMask[offset + tokenIndex] = BigInt(candidate.attentionMask[tokenIndex]);
|
| 352 |
+
tokenTypeIds[offset + tokenIndex] = BigInt(candidate.tokenTypeIds[tokenIndex]);
|
| 353 |
+
}
|
| 354 |
+
}
|
| 355 |
+
return { inputIds, attentionMask, tokenTypeIds, optionMask };
|
| 356 |
+
}
|
| 357 |
+
|
| 358 |
+
export function seededNoise(length: number, seed: number | undefined, questionIndex: number): Float32Array {
|
| 359 |
+
if (!Number.isInteger(length) || length < 1) throw new RangeError("noise length must be positive");
|
| 360 |
+
let state = ((seed ?? 0) ^ (questionIndex + 1) * 0x9e3779b9) >>> 0;
|
| 361 |
+
const result = new Float32Array(length);
|
| 362 |
+
const nextUnit = (): number => {
|
| 363 |
+
state = (Math.imul(state ^ (state >>> 16), 0x45d9f3b) + 0x27100001) >>> 0;
|
| 364 |
+
// Open interval (0, 1) avoids log(0) while keeping the stream deterministic.
|
| 365 |
+
return (state + 1) / 4294967297;
|
| 366 |
+
};
|
| 367 |
+
for (let index = 0; index < length; index += 1) {
|
| 368 |
+
const radius = Math.sqrt(-2 * Math.log(nextUnit()));
|
| 369 |
+
const angle = 2 * Math.PI * nextUnit();
|
| 370 |
+
result[index] = radius * Math.cos(angle);
|
| 371 |
+
if (index + 1 < length) {
|
| 372 |
+
result[index + 1] = radius * Math.sin(angle);
|
| 373 |
+
index += 1;
|
| 374 |
+
}
|
| 375 |
+
}
|
| 376 |
+
return result;
|
| 377 |
+
}
|
| 378 |
+
|
| 379 |
+
function sigmoid(value: number): number {
|
| 380 |
+
if (!Number.isFinite(value)) throw new Error("model returned a non-finite answerability logit");
|
| 381 |
+
return value >= 0 ? 1 / (1 + Math.exp(-value)) : Math.exp(value) / (1 + Math.exp(value));
|
| 382 |
+
}
|
| 383 |
+
|
| 384 |
+
function softmax(values: readonly number[]): number[] {
|
| 385 |
+
if (values.length === 0 || values.some((value) => Number.isNaN(value))) throw new Error("model returned invalid option scores");
|
| 386 |
+
const maximum = Math.max(...values);
|
| 387 |
+
if (!Number.isFinite(maximum)) throw new Error("model returned no finite option score");
|
| 388 |
+
const exponentials = values.map((value) => Math.exp(value - maximum));
|
| 389 |
+
const total = exponentials.reduce((sum, value) => sum + value, 0);
|
| 390 |
+
if (!Number.isFinite(total) || total <= 0) throw new Error("could not normalize model option scores");
|
| 391 |
+
return exponentials.map((value) => value / total);
|
| 392 |
+
}
|
| 393 |
+
|
| 394 |
+
function tensor(runtime: OrtModule, type: string, data: ArrayLike<number> | ArrayLike<bigint>, dims: readonly number[]): unknown {
|
| 395 |
+
return new runtime.Tensor(type, data, dims);
|
| 396 |
+
}
|
| 397 |
+
|
| 398 |
+
function tensorValues(output: OrtTensor | undefined, expectedLength: number, name: string): number[] {
|
| 399 |
+
if (!output || !output.dims) throw new Error(`missing output tensor: ${name}`);
|
| 400 |
+
const values = Array.from(output.data, Number);
|
| 401 |
+
if (values.length !== expectedLength) throw new Error(`output ${name} has ${values.length} values; expected ${expectedLength}`);
|
| 402 |
+
return values;
|
| 403 |
+
}
|
| 404 |
+
|
| 405 |
+
async function loadOrt(provider: OnnxExecutionProvider, options: OnnxWebBackendOptions): Promise<{ runtime: OrtModule; info: OnnxRuntimeInfo }> {
|
| 406 |
+
if (provider === "webgpu") {
|
| 407 |
+
const gpu = (globalThis.navigator as Navigator & { gpu?: GpuLike } | undefined)?.gpu;
|
| 408 |
+
if (!gpu) throw new Error("WebGPU is unavailable; choose wasm explicitly");
|
| 409 |
+
const adapter = await gpu.requestAdapter();
|
| 410 |
+
if (!adapter) throw new Error("WebGPU adapter request returned null");
|
| 411 |
+
await adapter.requestDevice();
|
| 412 |
+
}
|
| 413 |
+
const runtime = (provider === "webgpu"
|
| 414 |
+
? await import("onnxruntime-web/webgpu")
|
| 415 |
+
: await import("onnxruntime-web/wasm")) as unknown as OrtModule;
|
| 416 |
+
if (options.wasmPaths) runtime.env.wasm.wasmPaths = options.wasmPaths;
|
| 417 |
+
if (provider === "wasm") {
|
| 418 |
+
if (options.wasmNumThreads !== undefined) {
|
| 419 |
+
if (!Number.isInteger(options.wasmNumThreads) || options.wasmNumThreads < 1) throw new TypeError("wasmNumThreads must be a positive integer");
|
| 420 |
+
runtime.env.wasm.numThreads = options.wasmNumThreads;
|
| 421 |
+
}
|
| 422 |
+
}
|
| 423 |
+
return {
|
| 424 |
+
runtime,
|
| 425 |
+
info: {
|
| 426 |
+
provider,
|
| 427 |
+
ortVersion: runtime.env.versions?.web ?? runtime.env.versions?.common ?? null,
|
| 428 |
+
wasmNumThreads: provider === "wasm" ? runtime.env.wasm.numThreads ?? null : null,
|
| 429 |
+
webgpuAdapterRequested: provider === "webgpu",
|
| 430 |
+
webgpuDeviceRequested: provider === "webgpu",
|
| 431 |
+
},
|
| 432 |
+
};
|
| 433 |
+
}
|
| 434 |
+
|
| 435 |
+
export class OnnxWebBackend implements DecisionBackend {
|
| 436 |
+
readonly backend: Backend;
|
| 437 |
+
readonly runtimeInfo: OnnxRuntimeInfo;
|
| 438 |
+
readonly manifest: BundleManifest;
|
| 439 |
+
readonly manifestHash: string;
|
| 440 |
+
private readonly tokenizer: Tokenizer;
|
| 441 |
+
private readonly runtime: OrtModule;
|
| 442 |
+
private readonly session: OrtSession;
|
| 443 |
+
private readonly sequenceLength: number;
|
| 444 |
+
private readonly optionCount: number;
|
| 445 |
+
private readonly paddingId: number;
|
| 446 |
+
private readonly abstainThreshold: number;
|
| 447 |
+
private readonly answerabilityThreshold: number;
|
| 448 |
+
private readonly directOptionSlots: number | undefined;
|
| 449 |
+
private readonly initialNoise: OnnxWebBackendOptions["initialNoise"];
|
| 450 |
+
private readonly onTiming: OnnxWebBackendOptions["onTiming"];
|
| 451 |
+
private readonly onTrace: OnnxWebBackendOptions["onTrace"];
|
| 452 |
+
private disposed = false;
|
| 453 |
+
private constructor(assets: LoadedAssets, runtime: OrtModule, runtimeInfo: OnnxRuntimeInfo, session: OrtSession, options: OnnxWebBackendOptions) {
|
| 454 |
+
this.backend = assets.manifest.metadata.backend;
|
| 455 |
+
this.runtimeInfo = runtimeInfo;
|
| 456 |
+
this.manifest = assets.manifest;
|
| 457 |
+
this.manifestHash = assets.manifestHash;
|
| 458 |
+
this.tokenizer = assets.tokenizer;
|
| 459 |
+
this.runtime = runtime;
|
| 460 |
+
this.session = session;
|
| 461 |
+
this.sequenceLength = assets.manifest.metadata.sequence_length ?? 512;
|
| 462 |
+
this.optionCount = assets.manifest.metadata.option_count ?? 32;
|
| 463 |
+
this.paddingId = assets.tokenizer.token_to_id("[PAD]") ?? 0;
|
| 464 |
+
this.abstainThreshold = options.abstainThreshold ?? 0.55;
|
| 465 |
+
this.answerabilityThreshold = options.answerabilityThreshold ?? 0.5;
|
| 466 |
+
this.directOptionSlots = options.directOptionSlots;
|
| 467 |
+
this.initialNoise = options.initialNoise;
|
| 468 |
+
this.onTiming = options.onTiming;
|
| 469 |
+
this.onTrace = options.onTrace;
|
| 470 |
+
}
|
| 471 |
+
|
| 472 |
+
static async create(options: OnnxWebBackendOptions = {}): Promise<OnnxWebBackend> {
|
| 473 |
+
const assets = await loadAssets(options);
|
| 474 |
+
const provider = options.provider ?? "wasm";
|
| 475 |
+
assertFiniteUnit(options.abstainThreshold ?? 0.55, "abstainThreshold");
|
| 476 |
+
assertFiniteUnit(options.answerabilityThreshold ?? 0.5, "answerabilityThreshold");
|
| 477 |
+
const { runtime, info } = await loadOrt(provider, options);
|
| 478 |
+
const expectedInputs = assets.manifest.metadata.backend === "diffusion" ? REQUIRED_DIFFUSION_INPUTS : REQUIRED_DIRECT_INPUTS;
|
| 479 |
+
const modelInputs = assets.manifest.metadata.input_names ?? expectedInputs;
|
| 480 |
+
if (JSON.stringify(modelInputs) !== JSON.stringify(expectedInputs)) throw new Error("bundle input contract is not supported by this backend");
|
| 481 |
+
const outputNames = assets.manifest.metadata.output_names ?? (assets.manifest.metadata.backend === "diffusion" ? ["scores", "answerability"] : ["logits", "answerability"]);
|
| 482 |
+
const session = await runtime.InferenceSession.create(assets.model.buffer, {
|
| 483 |
+
executionProviders: [provider],
|
| 484 |
+
externalData: assets.externalData,
|
| 485 |
+
});
|
| 486 |
+
try {
|
| 487 |
+
if (session.inputNames.length !== expectedInputs.length || expectedInputs.some((name) => !session.inputNames.includes(name))) {
|
| 488 |
+
throw new Error(`loaded session inputs do not match the manifest: ${session.inputNames.join(",")}`);
|
| 489 |
+
}
|
| 490 |
+
if (outputNames.some((name) => !session.outputNames.includes(name))) throw new Error("loaded session outputs do not match the manifest");
|
| 491 |
+
} catch (error) {
|
| 492 |
+
await session.release();
|
| 493 |
+
throw error;
|
| 494 |
+
}
|
| 495 |
+
return new OnnxWebBackend(assets, runtime, info, session, options);
|
| 496 |
+
}
|
| 497 |
+
|
| 498 |
+
async decide(request: DecisionRequest): Promise<DecisionResponse> {
|
| 499 |
+
if (this.disposed) throw new Error("ONNX Runtime session has been disposed");
|
| 500 |
+
validateRequest(request);
|
| 501 |
+
const answers: QuestionAnswer[] = [];
|
| 502 |
+
for (const [questionIndex, question] of request.questions.entries()) {
|
| 503 |
+
const requestStarted = performance.now();
|
| 504 |
+
if (question.type === "score") {
|
| 505 |
+
throw new Error("score questions are not supported by this candidate-selection ONNX head");
|
| 506 |
+
}
|
| 507 |
+
const options = question.type === "boolean" && question.options.length === 0 ? ["true", "false"] : question.options;
|
| 508 |
+
const slots = this.backend === "diffusion" ? this.optionCount : this.directOptionSlots ?? options.length;
|
| 509 |
+
if (options.length === 0 || options.length > this.optionCount || slots > this.optionCount) {
|
| 510 |
+
throw new RangeError(`question ${question.id} has unsupported candidate count`);
|
| 511 |
+
}
|
| 512 |
+
const aggregateTokenCount = this.tokenizer.encode(candidateListText(request.state, question, options)).ids.length;
|
| 513 |
+
if (aggregateTokenCount > this.sequenceLength) {
|
| 514 |
+
throw new RangeError(`question ${question.id} exceeds the ${this.sequenceLength}-token aggregate input budget (tokens=${aggregateTokenCount})`);
|
| 515 |
+
}
|
| 516 |
+
const tokenizeStarted = performance.now();
|
| 517 |
+
const encoded = tokenizeCandidates(this.tokenizer, request.state, question, options, this.sequenceLength);
|
| 518 |
+
const tokenized = performance.now();
|
| 519 |
+
const padded = padCandidates(encoded, slots, this.sequenceLength, this.paddingId);
|
| 520 |
+
let traceNoise: Float32Array | null = null;
|
| 521 |
+
const feeds: Record<string, unknown> = {
|
| 522 |
+
input_ids: tensor(this.runtime, "int64", padded.inputIds, [1, slots, this.sequenceLength]),
|
| 523 |
+
attention_mask: tensor(this.runtime, "int64", padded.attentionMask, [1, slots, this.sequenceLength]),
|
| 524 |
+
token_type_ids: tensor(this.runtime, "int64", padded.tokenTypeIds, [1, slots, this.sequenceLength]),
|
| 525 |
+
option_mask: tensor(this.runtime, "bool", padded.optionMask, [1, slots]),
|
| 526 |
+
};
|
| 527 |
+
if (this.backend === "diffusion") {
|
| 528 |
+
const noise = this.initialNoise?.(this.optionCount, request.seed, questionIndex) ?? seededNoise(this.optionCount, request.seed, questionIndex);
|
| 529 |
+
if (!(noise instanceof Float32Array) || noise.length !== this.optionCount) throw new TypeError("initialNoise must return a Float32Array with option_count values");
|
| 530 |
+
traceNoise = noise;
|
| 531 |
+
feeds.initial_noise = tensor(this.runtime, "float32", noise, [1, this.optionCount]);
|
| 532 |
+
}
|
| 533 |
+
const inferenceStarted = performance.now();
|
| 534 |
+
const outputs = await this.session.run(feeds);
|
| 535 |
+
const readbackCompleted = performance.now();
|
| 536 |
+
const scoreName = this.backend === "diffusion" ? "scores" : "logits";
|
| 537 |
+
const scores = tensorValues(outputs[scoreName], slots, scoreName).slice(0, options.length);
|
| 538 |
+
const probabilities = softmax(scores);
|
| 539 |
+
const answerabilityLogit = tensorValues(outputs.answerability, 1, "answerability")[0];
|
| 540 |
+
const answerability = sigmoid(answerabilityLogit);
|
| 541 |
+
const choiceIndex = probabilities.reduce((best, value, index) => value > probabilities[best] ? index : best, 0);
|
| 542 |
+
const confidence = probabilities[choiceIndex] * answerability;
|
| 543 |
+
const abstainReason = answerability < this.answerabilityThreshold
|
| 544 |
+
? "answerability_below_threshold"
|
| 545 |
+
: confidence < this.abstainThreshold ? "confidence_below_threshold" : null;
|
| 546 |
+
const answer: QuestionAnswer = {
|
| 547 |
+
question_id: question.id,
|
| 548 |
+
choice: abstainReason ? null : options[choiceIndex],
|
| 549 |
+
probabilities: options.map((option, index) => ({ option, probability: probabilities[index] })),
|
| 550 |
+
confidence,
|
| 551 |
+
status: abstainReason ? "abstain" : "ok",
|
| 552 |
+
abstain_reason: abstainReason,
|
| 553 |
+
};
|
| 554 |
+
answers.push(answer);
|
| 555 |
+
this.onTrace?.({
|
| 556 |
+
questionId: question.id,
|
| 557 |
+
inputIds: Array.from(padded.inputIds, Number),
|
| 558 |
+
attentionMask: Array.from(padded.attentionMask, Number),
|
| 559 |
+
tokenTypeIds: Array.from(padded.tokenTypeIds, Number),
|
| 560 |
+
optionMask: Array.from(padded.optionMask, Number),
|
| 561 |
+
initialNoise: traceNoise === null ? null : Array.from(traceNoise),
|
| 562 |
+
rawScores: scores,
|
| 563 |
+
answerabilityLogit,
|
| 564 |
+
probabilities,
|
| 565 |
+
answer,
|
| 566 |
+
});
|
| 567 |
+
const postprocessCompleted = performance.now();
|
| 568 |
+
this.onTiming?.({
|
| 569 |
+
questionId: question.id,
|
| 570 |
+
live_candidates: encoded.length,
|
| 571 |
+
allocated_candidates: slots,
|
| 572 |
+
live_tokens: encoded.reduce((total, candidate) => total + candidate.inputIds.length, 0),
|
| 573 |
+
sequence_length: this.sequenceLength,
|
| 574 |
+
tokenizeMs: tokenized - tokenizeStarted,
|
| 575 |
+
prepareFeedMs: inferenceStarted - tokenized,
|
| 576 |
+
inferenceAndReadbackMs: readbackCompleted - inferenceStarted,
|
| 577 |
+
postprocessMs: postprocessCompleted - readbackCompleted,
|
| 578 |
+
totalRequestMs: postprocessCompleted - requestStarted,
|
| 579 |
+
});
|
| 580 |
+
}
|
| 581 |
+
const response: DecisionResponse = {
|
| 582 |
+
answers,
|
| 583 |
+
backend: this.backend,
|
| 584 |
+
model_id: this.manifest.metadata.model_id ?? `vons-${this.backend}-onnx-web`,
|
| 585 |
+
metadata: {
|
| 586 |
+
provider: this.runtimeInfo.provider,
|
| 587 |
+
ort_version: this.runtimeInfo.ortVersion,
|
| 588 |
+
sequence_length: this.sequenceLength,
|
| 589 |
+
candidate_slots: this.backend === "diffusion" ? this.optionCount : this.directOptionSlots ?? "live",
|
| 590 |
+
tokenizer: "@huggingface/tokenizers@0.2.0",
|
| 591 |
+
},
|
| 592 |
+
};
|
| 593 |
+
validateResponse(response);
|
| 594 |
+
return response;
|
| 595 |
+
}
|
| 596 |
+
|
| 597 |
+
async dispose(): Promise<void> {
|
| 598 |
+
if (this.disposed) return;
|
| 599 |
+
this.disposed = true;
|
| 600 |
+
await this.session.release();
|
| 601 |
+
}
|
| 602 |
+
}
|
| 603 |
+
|
| 604 |
+
export async function createOnnxWebBackend(options: OnnxWebBackendOptions = {}): Promise<OnnxWebBackend> {
|
| 605 |
+
return OnnxWebBackend.create(options);
|
| 606 |
+
}
|
sdk/typescript/test/index.test.mjs
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import assert from "node:assert/strict";
|
| 2 |
+
import test from "node:test";
|
| 3 |
+
|
| 4 |
+
const sdk = await import("../src/index.ts");
|
| 5 |
+
|
| 6 |
+
test("host policy refuses unregistered tools and requires exact high-risk consent", () => {
|
| 7 |
+
const adapter = new sdk.KASIAdapter({ unlock: "high" });
|
| 8 |
+
const first = { name: "unlock", arguments: { target: "lab" } };
|
| 9 |
+
const second = { name: "unlock", arguments: { target: "vault" } };
|
| 10 |
+
assert.equal(adapter.decide({ calls: [second], confidence: 0.9, risk: "low" }).action, "confirm");
|
| 11 |
+
assert.equal(adapter.decide({ calls: [second], confidence: 0.9, risk: "low" }, new Set([sdk.callFingerprint(first)])).action, "confirm");
|
| 12 |
+
assert.equal(adapter.decide({ calls: [second], confidence: 0.9, risk: "low" }, new Set([sdk.callFingerprint(second)])).action, "call");
|
| 13 |
+
assert.equal(new sdk.KASIAdapter({}).decide({ calls: [{ name: "read", arguments: {} }], confidence: 0.9, risk: "low" }).action, "refuse");
|
| 14 |
+
});
|
| 15 |
+
|
| 16 |
+
test("request and response validators enforce the shared contract", () => {
|
| 17 |
+
assert.doesNotThrow(() => sdk.validateRequest({
|
| 18 |
+
state: "ready",
|
| 19 |
+
questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["a", "b"] }],
|
| 20 |
+
}));
|
| 21 |
+
assert.throws(() => sdk.validateResponse({
|
| 22 |
+
answers: [{ question_id: "q", choice: "a", probabilities: [{ option: "a", probability: 0.5 }], confidence: 0.9, status: "ok" }],
|
| 23 |
+
backend: "direct",
|
| 24 |
+
model_id: "test",
|
| 25 |
+
}));
|
| 26 |
+
assert.throws(() => sdk.validateRequest({
|
| 27 |
+
state: "ready",
|
| 28 |
+
questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["a", "b"], rubric: ["bad"] }],
|
| 29 |
+
}), /only score question/);
|
| 30 |
+
assert.doesNotThrow(() => sdk.validateRequest({
|
| 31 |
+
state: "x".repeat(2000),
|
| 32 |
+
questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["a", "b"] }],
|
| 33 |
+
}));
|
| 34 |
+
assert.doesNotThrow(() => sdk.validateRequest({
|
| 35 |
+
state: "x".repeat(200),
|
| 36 |
+
questions: Array.from({ length: 8 }, (_, index) => ({
|
| 37 |
+
id: `q-${index}`,
|
| 38 |
+
type: "choice",
|
| 39 |
+
prompt: "Pick",
|
| 40 |
+
options: ["a", "b"],
|
| 41 |
+
})),
|
| 42 |
+
}));
|
| 43 |
+
assert.doesNotThrow(() => sdk.validateRequest({
|
| 44 |
+
state: "ready",
|
| 45 |
+
questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["x".repeat(2000), "b"] }],
|
| 46 |
+
}));
|
| 47 |
+
assert.throws(() => sdk.validateRequest({
|
| 48 |
+
state: "ready",
|
| 49 |
+
questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["", "b"] }],
|
| 50 |
+
}), /options must be strings/);
|
| 51 |
+
assert.doesNotThrow(() => sdk.validateRequest({
|
| 52 |
+
state: "ready",
|
| 53 |
+
questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["a", "b"] }],
|
| 54 |
+
}, 8, 512));
|
| 55 |
+
});
|
| 56 |
+
|
| 57 |
+
test("runtime selection is explicit", () => {
|
| 58 |
+
assert.equal(sdk.selectRuntime("wasm"), "wasm");
|
| 59 |
+
assert.throws(() => sdk.selectRuntime("webgpu"), /WebGPU is unavailable/);
|
| 60 |
+
const previousGpu = globalThis.gpu;
|
| 61 |
+
globalThis.gpu = {};
|
| 62 |
+
assert.equal(sdk.selectRuntime("webgpu"), "webgpu");
|
| 63 |
+
if (previousGpu === undefined) delete globalThis.gpu;
|
| 64 |
+
else globalThis.gpu = previousGpu;
|
| 65 |
+
});
|
sdk/typescript/test/onnx.test.mjs
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import assert from "node:assert/strict";
|
| 2 |
+
import test from "node:test";
|
| 3 |
+
|
| 4 |
+
const onnx = await import("../src/onnx.ts");
|
| 5 |
+
|
| 6 |
+
// Deliberately synthetic: public contract tests must not require pilot weights.
|
| 7 |
+
function testManifest() {
|
| 8 |
+
const record = (path, role, bytes = 4) => ({
|
| 9 |
+
path, role, bytes, sha256: "1".repeat(64), runtime_loaded: true, model_asset: true,
|
| 10 |
+
});
|
| 11 |
+
return {
|
| 12 |
+
schema_version: "vons.bundle.manifest/v1",
|
| 13 |
+
manifest: { path: "bundle-manifest.json", sha256: null, self_hash_excluded: true },
|
| 14 |
+
metadata: { backend: "direct", option_count: 2, sequence_length: 32 },
|
| 15 |
+
files: [
|
| 16 |
+
record("model.onnx", "model_graph"),
|
| 17 |
+
record("tokenizer.json", "tokenizer"),
|
| 18 |
+
record("tokenizer_config.json", "tokenizer"),
|
| 19 |
+
],
|
| 20 |
+
summary: { model_asset_bytes: 12, complete_download_bytes: 12 },
|
| 21 |
+
};
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
test("structured state and the training text template are deterministic", () => {
|
| 25 |
+
const question = { id: "q", type: "choice", prompt: "Pick", options: ["A"] };
|
| 26 |
+
assert.equal(onnx.serializeState({ z: 1, a: ["x", 2] }), '{"a":["x",2],"z":1}');
|
| 27 |
+
assert.equal(onnx.candidateText({ z: 1, a: ["x", 2] }, question, "A"), '{"a":["x",2],"z":1}\nQuestion: Pick\nCandidate: A');
|
| 28 |
+
});
|
| 29 |
+
|
| 30 |
+
test("default diffusion noise is deterministic Gaussian, not a bounded uniform surrogate", () => {
|
| 31 |
+
const first = onnx.seededNoise(32, 7, 0);
|
| 32 |
+
const second = onnx.seededNoise(32, 7, 0);
|
| 33 |
+
assert.deepEqual(Array.from(first), Array.from(second));
|
| 34 |
+
assert.ok(first.some((value) => value > 1 || value < -1));
|
| 35 |
+
assert.ok(Array.from(first).every(Number.isFinite));
|
| 36 |
+
});
|
| 37 |
+
|
| 38 |
+
test("tokenizer ids, masks, and overflow behavior are explicit", async () => {
|
| 39 |
+
const { Tokenizer } = await import("@huggingface/tokenizers");
|
| 40 |
+
const tokenizer = new Tokenizer(
|
| 41 |
+
{
|
| 42 |
+
version: "1.0",
|
| 43 |
+
added_tokens: [],
|
| 44 |
+
normalizer: { type: "BertNormalizer", clean_text: true, handle_chinese_chars: true, strip_accents: null, lowercase: true },
|
| 45 |
+
pre_tokenizer: { type: "BertPreTokenizer" },
|
| 46 |
+
post_processor: { type: "BertProcessing", sep: ["[SEP]", 2], cls: ["[CLS]", 1] },
|
| 47 |
+
decoder: { type: "WordPiece", prefix: "##", cleanup: true },
|
| 48 |
+
model: { type: "WordPiece", unk_token: "[UNK]", continuing_subword_prefix: "##", max_input_chars_per_word: 100,
|
| 49 |
+
vocab: { "[UNK]": 0, "[CLS]": 1, "[SEP]": 2, ready: 3, question: 4, pick: 5, candidate: 6, ":": 7, a: 8, b: 9, "/": 10, "?": 11, "中": 12, "文": 13 } },
|
| 50 |
+
},
|
| 51 |
+
{ tokenizer_class: "BertTokenizer", unk_token: "[UNK]", cls_token: "[CLS]", sep_token: "[SEP]", model_max_length: 512 },
|
| 52 |
+
);
|
| 53 |
+
const question = { id: "q", type: "choice", prompt: "Pick", options: ["A/B?", "中文"] };
|
| 54 |
+
const values = onnx.tokenizeCandidates(tokenizer, "ready", question, question.options, 512);
|
| 55 |
+
assert.deepEqual(values[0].inputIds, [1, 3, 4, 7, 5, 6, 7, 8, 10, 9, 11, 2]);
|
| 56 |
+
assert.deepEqual(values[1].inputIds, [1, 3, 4, 7, 5, 6, 7, 12, 13, 2]);
|
| 57 |
+
assert.equal(values[0].inputIds.length, values[0].attentionMask.length);
|
| 58 |
+
assert.deepEqual(new Set(values[0].attentionMask), new Set([1]));
|
| 59 |
+
assert.throws(() => onnx.tokenizeCandidates(tokenizer, "x", question, ["x".repeat(10000)], 4), /bundle limit/);
|
| 60 |
+
});
|
| 61 |
+
|
| 62 |
+
test("manifest integrity is checked before a runtime session is created", async () => {
|
| 63 |
+
const manifest = testManifest();
|
| 64 |
+
const original = manifest.files.find((item) => item.role === "model_graph");
|
| 65 |
+
original.sha256 = "0".repeat(64);
|
| 66 |
+
const fetch = async () => new Response(new Uint8Array(original.bytes), { status: 200 });
|
| 67 |
+
await assert.rejects(
|
| 68 |
+
onnx.createOnnxWebBackend({ manifest, baseUrl: "https://example.test/bundle/", fetch }),
|
| 69 |
+
/SHA-256 mismatch/,
|
| 70 |
+
);
|
| 71 |
+
});
|
| 72 |
+
|
| 73 |
+
test("release runs can pin manifest bytes and encoded traversal is rejected", async () => {
|
| 74 |
+
const manifest = testManifest();
|
| 75 |
+
await assert.rejects(
|
| 76 |
+
onnx.createOnnxWebBackend({
|
| 77 |
+
manifest,
|
| 78 |
+
baseUrl: "https://example.test/bundle/",
|
| 79 |
+
expectedManifestSha256: "0".repeat(64),
|
| 80 |
+
fetch: async () => new Response(new Uint8Array(), { status: 200 }),
|
| 81 |
+
}),
|
| 82 |
+
/manifest SHA-256 mismatch/,
|
| 83 |
+
);
|
| 84 |
+
const escaped = structuredClone(manifest);
|
| 85 |
+
escaped.files.find((item) => item.role === "model_graph").path = "%2e%2e/escape.onnx";
|
| 86 |
+
await assert.rejects(
|
| 87 |
+
onnx.createOnnxWebBackend({
|
| 88 |
+
manifest: escaped,
|
| 89 |
+
baseUrl: "https://example.test/bundle/",
|
| 90 |
+
fetch: async () => new Response(new Uint8Array(), { status: 200 }),
|
| 91 |
+
}),
|
| 92 |
+
/bundle asset path must be relative/,
|
| 93 |
+
);
|
| 94 |
+
});
|
| 95 |
+
|
| 96 |
+
test("malformed manifests are rejected before provider selection", async () => {
|
| 97 |
+
await assert.rejects(
|
| 98 |
+
onnx.createOnnxWebBackend({ manifest: { schema_version: "vons.bundle.manifest/v1" }, provider: "webgpu" }),
|
| 99 |
+
/metadata.backend|baseUrl|manifestUrl/,
|
| 100 |
+
);
|
| 101 |
+
});
|
sdk/typescript/test/parity.test.mjs
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import assert from "node:assert/strict";
|
| 2 |
+
import test from "node:test";
|
| 3 |
+
|
| 4 |
+
const { buildParityFixtures } = await import("../demo/parity.ts");
|
| 5 |
+
const { validateRequest } = await import("../src/index.ts");
|
| 6 |
+
|
| 7 |
+
test("parity matrix covers candidate counts, question types, and explicit overflow", () => {
|
| 8 |
+
const fixtures = buildParityFixtures();
|
| 9 |
+
assert.ok(fixtures.length >= 20);
|
| 10 |
+
assert.deepEqual(new Set(fixtures.map((item) => item.matrix.candidate_count)), new Set([2, 4, 8, 16, 32]));
|
| 11 |
+
assert.ok(fixtures.some((item) => item.request.questions[0]?.type === "boolean"));
|
| 12 |
+
assert.ok(fixtures.some((item) => item.request.questions[0]?.type === "score"));
|
| 13 |
+
assert.ok(fixtures.some((item) => item.expectedFailure));
|
| 14 |
+
assert.equal(new Set(fixtures.map((item) => item.id)).size, fixtures.length);
|
| 15 |
+
for (const fixture of fixtures.filter((item) => !item.expectedFailure)) {
|
| 16 |
+
assert.doesNotThrow(() => validateRequest(fixture.request), fixture.id);
|
| 17 |
+
}
|
| 18 |
+
});
|
tests/test_browser_benchmark.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib.util
|
| 2 |
+
import json
|
| 3 |
+
import tempfile
|
| 4 |
+
import unittest
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _load_tool():
|
| 9 |
+
path = Path(__file__).parents[1] / "tools" / "verify_browser_benchmark.py"
|
| 10 |
+
spec = importlib.util.spec_from_file_location("vons_browser_benchmark", path)
|
| 11 |
+
if spec is None or spec.loader is None:
|
| 12 |
+
raise RuntimeError("could not load browser benchmark verifier")
|
| 13 |
+
module = importlib.util.module_from_spec(spec)
|
| 14 |
+
spec.loader.exec_module(module)
|
| 15 |
+
return module
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
benchmark_tool = _load_tool()
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _sample(repeat: int, status: str = "ok") -> dict[str, object]:
|
| 22 |
+
values = {
|
| 23 |
+
"tokenizeMs": 1.0 + repeat,
|
| 24 |
+
"prepareFeedMs": 2.0 + repeat,
|
| 25 |
+
"inferenceAndReadbackMs": 3.0 + repeat,
|
| 26 |
+
"postprocessMs": 4.0 + repeat,
|
| 27 |
+
"totalRequestMs": 10.0 + repeat,
|
| 28 |
+
}
|
| 29 |
+
if status == "error":
|
| 30 |
+
values = {key: None for key in values}
|
| 31 |
+
return {
|
| 32 |
+
**values,
|
| 33 |
+
"session_start": 1,
|
| 34 |
+
"repeat": repeat,
|
| 35 |
+
"run_id": "run-1",
|
| 36 |
+
"case_id": f"case-{repeat + 1}",
|
| 37 |
+
"backend": "direct",
|
| 38 |
+
"requested_provider": "wasm",
|
| 39 |
+
"observed_provider": None,
|
| 40 |
+
"observed_provider_status": "session_created; execution_partition_not_exposed",
|
| 41 |
+
"live_candidates": 4 if status == "ok" else None,
|
| 42 |
+
"allocated_candidates": 4 if status == "ok" else None,
|
| 43 |
+
"live_tokens": 12 if status == "ok" else None,
|
| 44 |
+
"sequence_length": 512 if status == "ok" else None,
|
| 45 |
+
"bundle_hash": "a" * 64,
|
| 46 |
+
"tokenizer_hash": "b" * 64,
|
| 47 |
+
"input_hash": "c" * 64,
|
| 48 |
+
"noise_hash": None,
|
| 49 |
+
"status": status,
|
| 50 |
+
"js_heap_used_bytes_before": None,
|
| 51 |
+
"js_heap_used_bytes_after": None,
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class BrowserBenchmarkTests(unittest.TestCase):
|
| 56 |
+
def test_recomputes_raw_samples_and_allows_unavailable_memory(self) -> None:
|
| 57 |
+
report = {
|
| 58 |
+
"schema": "vons.browser-benchmark/v1",
|
| 59 |
+
"sessions": 1,
|
| 60 |
+
"warmup_excluded_per_session": 1,
|
| 61 |
+
"repeats_per_session": 3,
|
| 62 |
+
"run_id": "run-1",
|
| 63 |
+
"backend": "direct",
|
| 64 |
+
"cases": 1,
|
| 65 |
+
"requested_provider": "wasm",
|
| 66 |
+
"environment": {"user_agent": "test", "cross_origin_isolated": False},
|
| 67 |
+
"runtime_info": None,
|
| 68 |
+
"execution_provider_evidence": "test evidence",
|
| 69 |
+
"session_records": [{"session_start": 1, "load_ms": 5.0, "status": "ok"}],
|
| 70 |
+
"samples": [_sample(0), _sample(1), _sample(2, "error")],
|
| 71 |
+
}
|
| 72 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 73 |
+
path = Path(directory) / "report.json"
|
| 74 |
+
path.write_text(json.dumps(report), encoding="utf-8")
|
| 75 |
+
result = benchmark_tool.verify_report(path)
|
| 76 |
+
|
| 77 |
+
self.assertEqual(result["sample_rows"], 3)
|
| 78 |
+
self.assertEqual(result["successful_samples"], 2)
|
| 79 |
+
self.assertEqual(result["failed_samples"], 1)
|
| 80 |
+
self.assertEqual(result["recomputed_summary"]["total_request_ms_p50"], 10.5)
|
| 81 |
+
self.assertIsNone(result["recomputed_summary"]["js_heap_before_max"])
|
| 82 |
+
|
| 83 |
+
def test_rejects_failed_sample_with_fabricated_timing(self) -> None:
|
| 84 |
+
report = {
|
| 85 |
+
"schema": "vons.browser-benchmark/v1",
|
| 86 |
+
"sessions": 1,
|
| 87 |
+
"warmup_excluded_per_session": 1,
|
| 88 |
+
"repeats_per_session": 1,
|
| 89 |
+
"run_id": "run-1",
|
| 90 |
+
"backend": "direct",
|
| 91 |
+
"cases": 1,
|
| 92 |
+
"requested_provider": "webgpu",
|
| 93 |
+
"environment": {"user_agent": "test", "cross_origin_isolated": False},
|
| 94 |
+
"runtime_info": None,
|
| 95 |
+
"execution_provider_evidence": "test evidence",
|
| 96 |
+
"session_records": [{"session_start": 1, "load_ms": 5.0, "status": "ok"}],
|
| 97 |
+
"samples": [_sample(0, "error")],
|
| 98 |
+
}
|
| 99 |
+
report["samples"][0]["requested_provider"] = "webgpu"
|
| 100 |
+
report["samples"][0]["totalRequestMs"] = 2.0
|
| 101 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 102 |
+
path = Path(directory) / "report.json"
|
| 103 |
+
path.write_text(json.dumps(report), encoding="utf-8")
|
| 104 |
+
with self.assertRaises(ValueError):
|
| 105 |
+
benchmark_tool.verify_report(path)
|
| 106 |
+
|
| 107 |
+
def test_rejects_sample_without_provenance_hashes(self) -> None:
|
| 108 |
+
report = {
|
| 109 |
+
"schema": "vons.browser-benchmark/v1",
|
| 110 |
+
"sessions": 1,
|
| 111 |
+
"warmup_excluded_per_session": 1,
|
| 112 |
+
"repeats_per_session": 1,
|
| 113 |
+
"run_id": "run-1",
|
| 114 |
+
"backend": "direct",
|
| 115 |
+
"cases": 1,
|
| 116 |
+
"requested_provider": "wasm",
|
| 117 |
+
"environment": {"user_agent": "test", "cross_origin_isolated": False},
|
| 118 |
+
"runtime_info": None,
|
| 119 |
+
"execution_provider_evidence": "test evidence",
|
| 120 |
+
"session_records": [{"session_start": 1, "load_ms": 5.0, "status": "ok"}],
|
| 121 |
+
"samples": [_sample(0)],
|
| 122 |
+
}
|
| 123 |
+
report["samples"][0]["input_hash"] = None
|
| 124 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 125 |
+
path = Path(directory) / "report.json"
|
| 126 |
+
path.write_text(json.dumps(report), encoding="utf-8")
|
| 127 |
+
with self.assertRaises(ValueError):
|
| 128 |
+
benchmark_tool.verify_report(path)
|
| 129 |
+
|
| 130 |
+
def test_rejects_partial_session_matrix_by_default(self) -> None:
|
| 131 |
+
report = {
|
| 132 |
+
"schema": "vons.browser-benchmark/v1",
|
| 133 |
+
"sessions": 2,
|
| 134 |
+
"warmup_excluded_per_session": 1,
|
| 135 |
+
"repeats_per_session": 1,
|
| 136 |
+
"run_id": "run-1",
|
| 137 |
+
"backend": "direct",
|
| 138 |
+
"cases": 1,
|
| 139 |
+
"requested_provider": "wasm",
|
| 140 |
+
"environment": {"user_agent": "test", "cross_origin_isolated": False},
|
| 141 |
+
"runtime_info": None,
|
| 142 |
+
"execution_provider_evidence": "test evidence",
|
| 143 |
+
"session_records": [
|
| 144 |
+
{"session_start": 1, "load_ms": 5.0, "status": "ok"},
|
| 145 |
+
{"session_start": 2, "load_ms": 1.0, "status": "error", "error_kind": "fetch"},
|
| 146 |
+
],
|
| 147 |
+
"samples": [_sample(0)],
|
| 148 |
+
}
|
| 149 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 150 |
+
path = Path(directory) / "report.json"
|
| 151 |
+
path.write_text(json.dumps(report), encoding="utf-8")
|
| 152 |
+
with self.assertRaises(ValueError):
|
| 153 |
+
benchmark_tool.verify_report(path)
|
| 154 |
+
result = benchmark_tool.verify_report(path, allow_incomplete=True)
|
| 155 |
+
|
| 156 |
+
self.assertFalse(result["complete"])
|
| 157 |
+
self.assertTrue(result["incomplete_reasons"])
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
if __name__ == "__main__":
|
| 161 |
+
unittest.main()
|
tests/test_browser_benchmark_summary.py
ADDED
|
@@ -0,0 +1,1130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for tools/summarize_browser_benchmark.py.
|
| 2 |
+
|
| 3 |
+
Covers:
|
| 4 |
+
- Exact expected linearly interpolated percentiles (p50, p95) for known datasets
|
| 5 |
+
- Null/failed/timeout row preservation with nullable timing slots
|
| 6 |
+
- Malformed rows, non-finite timings, wrong types, bad SHA256
|
| 7 |
+
- Homogeneous condition enforcement and mixed-condition rejection
|
| 8 |
+
- Duplicate (run_id, session_start, repeat) identity rejection
|
| 9 |
+
- Insufficient-observation completeness flag without error
|
| 10 |
+
- CLI behavior via subprocess on a small valid fixture
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import json
|
| 16 |
+
import math
|
| 17 |
+
import subprocess
|
| 18 |
+
import sys
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from typing import Any
|
| 21 |
+
|
| 22 |
+
import pytest
|
| 23 |
+
|
| 24 |
+
TOOLS_DIR = Path(__file__).resolve().parent.parent / "tools"
|
| 25 |
+
SUMMARY_TOOL = TOOLS_DIR / "summarize_browser_benchmark.py"
|
| 26 |
+
|
| 27 |
+
sys.path.insert(0, str(TOOLS_DIR))
|
| 28 |
+
from summarize_browser_benchmark import ( # noqa: E402, RUF100
|
| 29 |
+
TIMING_FIELDS,
|
| 30 |
+
_validate_row,
|
| 31 |
+
percentile,
|
| 32 |
+
validate_and_aggregate,
|
| 33 |
+
validate_and_aggregate_report,
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
SHA_A = "a" * 64
|
| 37 |
+
SHA_B = "b" * 64
|
| 38 |
+
SHA_C = "c" * 64
|
| 39 |
+
SHA_D = "d" * 64
|
| 40 |
+
SHA_NOISE = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
| 41 |
+
|
| 42 |
+
BASE_ENV = {
|
| 43 |
+
"user_agent": "TestBrowser/1.0",
|
| 44 |
+
"cross_origin_isolated": True,
|
| 45 |
+
"platform": "Test OS",
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _provenance(*, warmup: int = 10, semantics: str = "page_reload") -> dict[str, Any]:
|
| 50 |
+
return {
|
| 51 |
+
"environment": dict(BASE_ENV),
|
| 52 |
+
"execution_provider_evidence": "window.ort.env.backend='wasm'",
|
| 53 |
+
"warmup_excluded_per_session": warmup,
|
| 54 |
+
"session_start_semantics": semantics,
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _row(
|
| 59 |
+
*,
|
| 60 |
+
session_start: int = 1,
|
| 61 |
+
repeat: int = 0,
|
| 62 |
+
case_id: str = "case-0",
|
| 63 |
+
backend: str = "direct",
|
| 64 |
+
status: str = "ok",
|
| 65 |
+
error: str | None = None,
|
| 66 |
+
timings: dict[str, float] | None = None,
|
| 67 |
+
candidate_count: int = 4,
|
| 68 |
+
padded_option_slots: int = 8,
|
| 69 |
+
sequence_length: int = 64,
|
| 70 |
+
seed: int = 42,
|
| 71 |
+
noise_hash: str | None = None,
|
| 72 |
+
heap_before: float | None = None,
|
| 73 |
+
heap_after: float | None = None,
|
| 74 |
+
observed_provider: str = "wasm",
|
| 75 |
+
) -> dict[str, Any]:
|
| 76 |
+
if backend == "diffusion" and noise_hash is None:
|
| 77 |
+
noise_hash = SHA_NOISE
|
| 78 |
+
ok_timings: dict[str, float] = (
|
| 79 |
+
timings
|
| 80 |
+
if timings is not None
|
| 81 |
+
else {
|
| 82 |
+
"tokenizeMs": 1.0,
|
| 83 |
+
"prepareFeedMs": 0.5,
|
| 84 |
+
"inferenceAndReadbackMs": 10.0,
|
| 85 |
+
"postprocessMs": 0.25,
|
| 86 |
+
"totalRequestMs": 11.75,
|
| 87 |
+
}
|
| 88 |
+
)
|
| 89 |
+
row: dict[str, Any] = {
|
| 90 |
+
"run_id": "run-alpha",
|
| 91 |
+
"session_start": session_start,
|
| 92 |
+
"repeat": repeat,
|
| 93 |
+
"case_id": case_id,
|
| 94 |
+
"backend": backend,
|
| 95 |
+
"requested_provider": "wasm",
|
| 96 |
+
"observed_provider": observed_provider,
|
| 97 |
+
"bundle_hash": SHA_A,
|
| 98 |
+
"tokenizer_hash": SHA_B,
|
| 99 |
+
"input_hash": SHA_C,
|
| 100 |
+
"noise_hash": noise_hash,
|
| 101 |
+
"candidate_count": candidate_count,
|
| 102 |
+
"padded_option_slots": padded_option_slots,
|
| 103 |
+
"sequence_length": sequence_length,
|
| 104 |
+
"seed": seed,
|
| 105 |
+
"ort_version": "1.20.1",
|
| 106 |
+
"status": status,
|
| 107 |
+
"error": error,
|
| 108 |
+
"provenance": _provenance(),
|
| 109 |
+
}
|
| 110 |
+
if status == "ok":
|
| 111 |
+
row.update(ok_timings)
|
| 112 |
+
if heap_before is not None:
|
| 113 |
+
row["js_heap_used_bytes_before"] = heap_before
|
| 114 |
+
if heap_after is not None:
|
| 115 |
+
row["js_heap_used_bytes_after"] = heap_after
|
| 116 |
+
else:
|
| 117 |
+
for field in TIMING_FIELDS:
|
| 118 |
+
row[field] = None
|
| 119 |
+
if timings is not None:
|
| 120 |
+
for field, value in timings.items():
|
| 121 |
+
row[field] = value
|
| 122 |
+
if error is None:
|
| 123 |
+
row["error"] = "simulated failure"
|
| 124 |
+
return row
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _write_jsonl(tmp_path: Path, rows: list[dict[str, Any]], *, name: str = "input.jsonl") -> Path:
|
| 128 |
+
path = tmp_path / name
|
| 129 |
+
with path.open("w", encoding="utf-8") as handle:
|
| 130 |
+
for row in rows:
|
| 131 |
+
handle.write(json.dumps(row, sort_keys=True) + "\n")
|
| 132 |
+
return path
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
# ---------------------------------------------------------------------------
|
| 136 |
+
# percentile linear interpolation correctness
|
| 137 |
+
# ---------------------------------------------------------------------------
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
class TestPercentile:
|
| 141 |
+
def test_empty_is_null(self) -> None:
|
| 142 |
+
assert percentile([], 0.5) is None
|
| 143 |
+
|
| 144 |
+
def test_single_value(self) -> None:
|
| 145 |
+
assert percentile([7.0], 0.5) == 7.0
|
| 146 |
+
|
| 147 |
+
def test_two_values_p50_is_midpoint(self) -> None:
|
| 148 |
+
# index = (2-1)*0.5 = 0.5 -> lower=0, upper=1 -> midpoint
|
| 149 |
+
assert percentile([10.0, 20.0], 0.5) == pytest.approx(15.0)
|
| 150 |
+
|
| 151 |
+
def test_two_values_p95_near_upper(self) -> None:
|
| 152 |
+
# index = 1 * 0.95 = 0.95 -> 10 + 10*0.95 = 19.5
|
| 153 |
+
assert percentile([10.0, 20.0], 0.95) == pytest.approx(19.5)
|
| 154 |
+
|
| 155 |
+
def test_four_values_p50_exact_interpolation(self) -> None:
|
| 156 |
+
# sorted: [1,2,3,4]; index = (4-1)*0.5 = 1.5
|
| 157 |
+
# lower=1 -> 2, upper=2 -> 3, fraction=0.5 -> 2 + 1*0.5 = 2.5
|
| 158 |
+
assert percentile([3.0, 1.0, 4.0, 2.0], 0.5) == pytest.approx(2.5)
|
| 159 |
+
|
| 160 |
+
def test_eight_values_p95(self) -> None:
|
| 161 |
+
values = [float(x) for x in range(1, 9)]
|
| 162 |
+
# sorted: [1..8]; index = (8-1)*0.95 = 7*0.95 = 6.65
|
| 163 |
+
# lower=6 -> 7.0, upper=7 -> 8.0, delta=0.65 -> 7.0 + 1.0*0.65 = 7.65
|
| 164 |
+
assert percentile(values, 0.95) == pytest.approx(7.65)
|
| 165 |
+
|
| 166 |
+
def test_fraction_out_of_range_rejected(self) -> None:
|
| 167 |
+
with pytest.raises(ValueError):
|
| 168 |
+
percentile([1.0, 2.0], 1.5)
|
| 169 |
+
with pytest.raises(ValueError):
|
| 170 |
+
percentile([1.0, 2.0], -0.1)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
# ---------------------------------------------------------------------------
|
| 174 |
+
# Row validation
|
| 175 |
+
# ---------------------------------------------------------------------------
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
class TestRowValidation:
|
| 179 |
+
def test_missing_status_rejected(self) -> None:
|
| 180 |
+
row = _row()
|
| 181 |
+
del row["status"]
|
| 182 |
+
with pytest.raises(ValueError, match="missing required field status"):
|
| 183 |
+
_validate_row(row, 1)
|
| 184 |
+
|
| 185 |
+
def test_invalid_status_string_rejected(self) -> None:
|
| 186 |
+
row = _row(status="pending")
|
| 187 |
+
with pytest.raises(ValueError, match="status must be ok, error, or timeout"):
|
| 188 |
+
_validate_row(row, 1)
|
| 189 |
+
|
| 190 |
+
def test_ok_row_with_error_value_rejected(self) -> None:
|
| 191 |
+
row = _row(status="ok", error="should be empty")
|
| 192 |
+
with pytest.raises(ValueError, match="error must be null for successful"):
|
| 193 |
+
_validate_row(row, 1)
|
| 194 |
+
|
| 195 |
+
def test_failed_row_with_partial_timings_allowed(self) -> None:
|
| 196 |
+
row = _row(status="error", timings={"totalRequestMs": 500.0, "tokenizeMs": 10.0})
|
| 197 |
+
assert row["status"] == "error"
|
| 198 |
+
assert row["totalRequestMs"] == 500.0
|
| 199 |
+
assert row["tokenizeMs"] == 10.0
|
| 200 |
+
assert row["prepareFeedMs"] is None
|
| 201 |
+
_validate_row(row, 1)
|
| 202 |
+
|
| 203 |
+
def test_successful_row_missing_timing_rejected(self) -> None:
|
| 204 |
+
row = _row()
|
| 205 |
+
del row["inferenceAndReadbackMs"]
|
| 206 |
+
with pytest.raises(ValueError, match="successful sample requires inferenceAndReadbackMs"):
|
| 207 |
+
_validate_row(row, 1)
|
| 208 |
+
|
| 209 |
+
def test_nonfinite_timing_rejected(self) -> None:
|
| 210 |
+
row = _row(timings={"tokenizeMs": math.nan, "prepareFeedMs": 0.5,
|
| 211 |
+
"inferenceAndReadbackMs": 10.0, "postprocessMs": 0.25,
|
| 212 |
+
"totalRequestMs": 11.75})
|
| 213 |
+
with pytest.raises((ValueError, TypeError), match="finite"):
|
| 214 |
+
_validate_row(row, 1)
|
| 215 |
+
|
| 216 |
+
def test_negative_timing_rejected(self) -> None:
|
| 217 |
+
row = _row(timings={"tokenizeMs": -1.0, "prepareFeedMs": 0.5,
|
| 218 |
+
"inferenceAndReadbackMs": 10.0, "postprocessMs": 0.25,
|
| 219 |
+
"totalRequestMs": 11.75})
|
| 220 |
+
with pytest.raises(ValueError, match="finite non-negative"):
|
| 221 |
+
_validate_row(row, 1)
|
| 222 |
+
|
| 223 |
+
def test_invalid_sha256_rejected(self) -> None:
|
| 224 |
+
row = _row()
|
| 225 |
+
row["bundle_hash"] = "short-and-wrong"
|
| 226 |
+
with pytest.raises(ValueError, match="SHA-256 hex string"):
|
| 227 |
+
_validate_row(row, 1)
|
| 228 |
+
|
| 229 |
+
def test_uppercase_sha256_rejected(self) -> None:
|
| 230 |
+
row = _row()
|
| 231 |
+
row["bundle_hash"] = "A" * 64
|
| 232 |
+
with pytest.raises(ValueError, match="lowercase SHA-256"):
|
| 233 |
+
_validate_row(row, 1)
|
| 234 |
+
|
| 235 |
+
def test_non_positive_shape_rejected(self) -> None:
|
| 236 |
+
row = _row(candidate_count=0)
|
| 237 |
+
with pytest.raises(ValueError, match="positive integer"):
|
| 238 |
+
_validate_row(row, 1)
|
| 239 |
+
row2 = _row(padded_option_slots=-5)
|
| 240 |
+
with pytest.raises(ValueError, match="positive integer"):
|
| 241 |
+
_validate_row(row2, 2)
|
| 242 |
+
row3 = _row(sequence_length=0)
|
| 243 |
+
with pytest.raises(ValueError, match="positive integer"):
|
| 244 |
+
_validate_row(row3, 3)
|
| 245 |
+
|
| 246 |
+
def test_diffusion_noise_required(self) -> None:
|
| 247 |
+
row = _row(backend="diffusion", noise_hash=None)
|
| 248 |
+
row["noise_hash"] = None
|
| 249 |
+
with pytest.raises(ValueError, match="noise_hash is required for diffusion"):
|
| 250 |
+
_validate_row(row, 1)
|
| 251 |
+
|
| 252 |
+
def test_repeat_negative_rejected(self) -> None:
|
| 253 |
+
row = _row(repeat=-1)
|
| 254 |
+
with pytest.raises(ValueError, match="repeat must be a non-negative integer"):
|
| 255 |
+
_validate_row(row, 1)
|
| 256 |
+
|
| 257 |
+
def test_requested_provider_must_be_wasm_or_webgpu(self) -> None:
|
| 258 |
+
row = _row()
|
| 259 |
+
row["requested_provider"] = "cpu"
|
| 260 |
+
with pytest.raises(ValueError, match="requested_provider must be wasm or webgpu"):
|
| 261 |
+
_validate_row(row, 1)
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
# ---------------------------------------------------------------------------
|
| 265 |
+
# End-to-end aggregation with actual expected percentiles
|
| 266 |
+
# ---------------------------------------------------------------------------
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def _build_120row_fixture(
|
| 270 |
+
*,
|
| 271 |
+
backend: str = "direct",
|
| 272 |
+
extra_failures: int = 0,
|
| 273 |
+
warmup: int = 10,
|
| 274 |
+
) -> list[dict[str, Any]]:
|
| 275 |
+
"""10 sessions x 12 repeats = 120 rows with deterministic increasing timings.
|
| 276 |
+
|
| 277 |
+
totalRequestMs values for successful rows = 1.0, 2.0, ..., N_success.0.
|
| 278 |
+
p50 (120 successes, all ok):
|
| 279 |
+
index = (120-1)*0.5 = 59.5 -> values[59]=60.0, values[60]=61.0, frac=0.5 -> 60.5
|
| 280 |
+
p95:
|
| 281 |
+
index = 119*0.95 = 113.05 -> values[113]=114.0, values[114]=115.0, frac=0.05
|
| 282 |
+
-> 114.0 + 1.0*0.05 = 114.05
|
| 283 |
+
"""
|
| 284 |
+
rows: list[dict[str, Any]] = []
|
| 285 |
+
counter = 0
|
| 286 |
+
for session in range(1, 11):
|
| 287 |
+
for repeat in range(12):
|
| 288 |
+
counter += 1
|
| 289 |
+
if extra_failures > 0 and counter % 15 == 0:
|
| 290 |
+
rows.append(
|
| 291 |
+
_row(
|
| 292 |
+
session_start=session,
|
| 293 |
+
repeat=repeat,
|
| 294 |
+
case_id=f"case-{counter}",
|
| 295 |
+
backend=backend,
|
| 296 |
+
status="error",
|
| 297 |
+
error="simulated",
|
| 298 |
+
)
|
| 299 |
+
)
|
| 300 |
+
extra_failures -= 1
|
| 301 |
+
else:
|
| 302 |
+
total = float(counter)
|
| 303 |
+
tokenize = 0.1 * counter
|
| 304 |
+
prepare = 0.05 * counter
|
| 305 |
+
inference = 0.8 * counter
|
| 306 |
+
post = 0.05 * counter
|
| 307 |
+
rows.append(
|
| 308 |
+
_row(
|
| 309 |
+
session_start=session,
|
| 310 |
+
repeat=repeat,
|
| 311 |
+
case_id=f"case-{counter}",
|
| 312 |
+
backend=backend,
|
| 313 |
+
status="ok",
|
| 314 |
+
timings={
|
| 315 |
+
"tokenizeMs": tokenize,
|
| 316 |
+
"prepareFeedMs": prepare,
|
| 317 |
+
"inferenceAndReadbackMs": inference,
|
| 318 |
+
"postprocessMs": post,
|
| 319 |
+
"totalRequestMs": total,
|
| 320 |
+
},
|
| 321 |
+
heap_before=1000.0 + counter,
|
| 322 |
+
heap_after=2000.0 + counter,
|
| 323 |
+
)
|
| 324 |
+
)
|
| 325 |
+
rows[-1]["provenance"] = _provenance(warmup=warmup)
|
| 326 |
+
return rows
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
class TestAggregationPercentiles:
|
| 330 |
+
def test_120rows_percentiles_match_exact_expected(self, tmp_path: Path) -> None:
|
| 331 |
+
rows = _build_120row_fixture()
|
| 332 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 333 |
+
summary = validate_and_aggregate(input_path)
|
| 334 |
+
|
| 335 |
+
assert summary["sample_counts"]["successful"] == 120
|
| 336 |
+
assert summary["sample_counts"]["failed"] == 0
|
| 337 |
+
assert summary["sample_counts"]["unique_session_starts"] == 10
|
| 338 |
+
assert summary["measurement_completeness"]["complete"] is True
|
| 339 |
+
|
| 340 |
+
p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"]
|
| 341 |
+
p95 = summary["stage_percentiles_ms"]["totalRequestMs"]["p95"]
|
| 342 |
+
assert p50 == pytest.approx(60.5)
|
| 343 |
+
assert p95 == pytest.approx(114.05)
|
| 344 |
+
|
| 345 |
+
# tokenizeMs values = 0.1 * counter for counter=1..120
|
| 346 |
+
# sorted values are 0.1, 0.2, ..., 12.0
|
| 347 |
+
# p50: index=59.5 -> 6.0 + (6.1-6.0)*0.5 = 6.05
|
| 348 |
+
assert summary["stage_percentiles_ms"]["tokenizeMs"]["p50"] == pytest.approx(6.05)
|
| 349 |
+
# p95: index=113.05 -> values[113]=11.4, values[114]=11.5, frac=0.05 -> 11.405
|
| 350 |
+
assert summary["stage_percentiles_ms"]["tokenizeMs"]["p95"] == pytest.approx(11.405)
|
| 351 |
+
|
| 352 |
+
def test_failed_rows_counted_timings_nullable(self, tmp_path: Path) -> None:
|
| 353 |
+
rows = _build_120row_fixture(extra_failures=8)
|
| 354 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 355 |
+
summary = validate_and_aggregate(input_path)
|
| 356 |
+
|
| 357 |
+
assert summary["sample_counts"]["successful"] == 112
|
| 358 |
+
assert summary["sample_counts"]["failed"] == 8
|
| 359 |
+
assert summary["measurement_completeness"]["complete"] is True
|
| 360 |
+
|
| 361 |
+
# 112 successes: totalRequestMs are the 112 counter values that
|
| 362 |
+
# are not divisible by 15. Counters divisible by 15 (15,30,...,105,120)
|
| 363 |
+
# are failures. Total counters divisible by 15 up to 120: 120//15=8.
|
| 364 |
+
# totalRequestMs values are 1..14,16..29,...,106..119 (8 missing
|
| 365 |
+
# values: 15,30,45,60,75,90,105,120). All increasing order is still
|
| 366 |
+
# preserved because values increase monotonically with counter and
|
| 367 |
+
# only the multiples of 15 are removed.
|
| 368 |
+
#
|
| 369 |
+
# For percentile of 112 values:
|
| 370 |
+
# p50 index = (112-1)*0.5 = 55.5 -> values[55] and values[56]
|
| 371 |
+
# Before each multiple-of-15 threshold, 1 value has been skipped per
|
| 372 |
+
# prior group. Which values are at index 55/56?
|
| 373 |
+
# For each group of 15 counters, 14 survive. Group 1 (1-14) -> indices 0..13 (14).
|
| 374 |
+
# Group 2 (16-29) -> indices 14..27 (14, cumulative 28).
|
| 375 |
+
# Group 3 (31-44) -> indices 28..41 (14, cumulative 42).
|
| 376 |
+
# Group 4 (46-59) -> indices 42..55 (14, cumulative 56).
|
| 377 |
+
# So index 55 = last of group 4 -> counter 59 (value 59.0).
|
| 378 |
+
# Index 56 = first of group 5 -> counter 61 (value 61.0).
|
| 379 |
+
# Interpolation at 55.5 -> 0.5 fraction: 59.0 + 2.0 * 0.5 = 60.0
|
| 380 |
+
p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"]
|
| 381 |
+
assert p50 == pytest.approx(60.0)
|
| 382 |
+
|
| 383 |
+
def test_heap_maxima_from_observations_only(self, tmp_path: Path) -> None:
|
| 384 |
+
rows = _build_120row_fixture()
|
| 385 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 386 |
+
summary = validate_and_aggregate(input_path)
|
| 387 |
+
# counter 1..120 -> heap_before = 1000 + counter -> max at 1120
|
| 388 |
+
# counter 1..120 -> heap_after = 2000 + counter -> max at 2120
|
| 389 |
+
assert summary["heap_maxima_bytes"]["js_heap_used_bytes_before"] == pytest.approx(1120.0)
|
| 390 |
+
assert summary["heap_maxima_bytes"]["js_heap_used_bytes_after"] == pytest.approx(2120.0)
|
| 391 |
+
|
| 392 |
+
def test_all_null_heap_yields_null_maxima(self, tmp_path: Path) -> None:
|
| 393 |
+
rows = _build_120row_fixture()
|
| 394 |
+
for row in rows:
|
| 395 |
+
if row["status"] == "ok":
|
| 396 |
+
row.pop("js_heap_used_bytes_before", None)
|
| 397 |
+
row.pop("js_heap_used_bytes_after", None)
|
| 398 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 399 |
+
summary = validate_and_aggregate(input_path)
|
| 400 |
+
assert summary["heap_maxima_bytes"]["js_heap_used_bytes_before"] is None
|
| 401 |
+
assert summary["heap_maxima_bytes"]["js_heap_used_bytes_after"] is None
|
| 402 |
+
|
| 403 |
+
|
| 404 |
+
# ---------------------------------------------------------------------------
|
| 405 |
+
# Empty input, duplicates, mixed conditions
|
| 406 |
+
# ---------------------------------------------------------------------------
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
class TestValidationFailuresAggregate:
|
| 410 |
+
def test_empty_input_raises(self, tmp_path: Path) -> None:
|
| 411 |
+
path = tmp_path / "empty.jsonl"
|
| 412 |
+
path.write_text("", encoding="utf-8")
|
| 413 |
+
with pytest.raises(ValueError, match="no samples"):
|
| 414 |
+
validate_and_aggregate(path)
|
| 415 |
+
|
| 416 |
+
def test_duplicate_identity_raises(self, tmp_path: Path) -> None:
|
| 417 |
+
rows = [
|
| 418 |
+
_row(session_start=1, repeat=0, case_id="a"),
|
| 419 |
+
_row(session_start=1, repeat=0, case_id="b"),
|
| 420 |
+
]
|
| 421 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 422 |
+
with pytest.raises(ValueError, match="duplicate.*identity"):
|
| 423 |
+
validate_and_aggregate(input_path)
|
| 424 |
+
|
| 425 |
+
def test_mixed_bundle_hash_raises(self, tmp_path: Path) -> None:
|
| 426 |
+
rows = [_row(session_start=1, repeat=0), _row(session_start=1, repeat=1)]
|
| 427 |
+
rows[1]["bundle_hash"] = SHA_D
|
| 428 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 429 |
+
with pytest.raises(ValueError, match="mixed experimental conditions"):
|
| 430 |
+
validate_and_aggregate(input_path)
|
| 431 |
+
|
| 432 |
+
def test_mixed_backend_raises(self, tmp_path: Path) -> None:
|
| 433 |
+
rows = [
|
| 434 |
+
_row(session_start=1, repeat=0, backend="direct"),
|
| 435 |
+
_row(session_start=1, repeat=1, backend="diffusion", noise_hash=SHA_NOISE),
|
| 436 |
+
]
|
| 437 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 438 |
+
with pytest.raises(ValueError, match="mixed experimental conditions"):
|
| 439 |
+
validate_and_aggregate(input_path)
|
| 440 |
+
|
| 441 |
+
def test_mixed_environment_raises(self, tmp_path: Path) -> None:
|
| 442 |
+
rows = [_row(session_start=1, repeat=0), _row(session_start=1, repeat=1)]
|
| 443 |
+
rows[1]["provenance"] = _provenance()
|
| 444 |
+
rows[1]["provenance"]["environment"]["platform"] = "Different OS"
|
| 445 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 446 |
+
with pytest.raises(ValueError, match="mixed experimental conditions"):
|
| 447 |
+
validate_and_aggregate(input_path)
|
| 448 |
+
|
| 449 |
+
def test_mixed_warmup_raises(self, tmp_path: Path) -> None:
|
| 450 |
+
rows = [_row(session_start=1, repeat=0), _row(session_start=1, repeat=1)]
|
| 451 |
+
rows[1]["provenance"] = _provenance(warmup=20)
|
| 452 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 453 |
+
with pytest.raises(ValueError, match="mixed experimental conditions"):
|
| 454 |
+
validate_and_aggregate(input_path)
|
| 455 |
+
|
| 456 |
+
def test_diffusion_backend_homogeneous(self, tmp_path: Path) -> None:
|
| 457 |
+
"""Diffusion samples must carry noise_hash; varying noise_hash by row is allowed."""
|
| 458 |
+
rows = []
|
| 459 |
+
for session in range(1, 7):
|
| 460 |
+
for repeat in range(20):
|
| 461 |
+
idx = session * 100 + repeat
|
| 462 |
+
rows.append(
|
| 463 |
+
_row(
|
| 464 |
+
session_start=session,
|
| 465 |
+
repeat=repeat,
|
| 466 |
+
case_id=f"case-{idx}",
|
| 467 |
+
backend="diffusion",
|
| 468 |
+
noise_hash=f"{idx:x}".zfill(64),
|
| 469 |
+
)
|
| 470 |
+
)
|
| 471 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 472 |
+
summary = validate_and_aggregate(input_path)
|
| 473 |
+
assert summary["experimental_condition"]["backend"] == "diffusion"
|
| 474 |
+
assert summary["sample_counts"]["successful"] == 120
|
| 475 |
+
assert summary["sample_counts"]["failed"] == 0
|
| 476 |
+
|
| 477 |
+
|
| 478 |
+
# ---------------------------------------------------------------------------
|
| 479 |
+
# Completeness flag and provenance semantics
|
| 480 |
+
# ---------------------------------------------------------------------------
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
class TestCompletenessAndProvenance:
|
| 484 |
+
def test_insufficient_sessions_reported_incomplete(self, tmp_path: Path) -> None:
|
| 485 |
+
rows = []
|
| 486 |
+
for session in range(1, 4): # only 3 sessions
|
| 487 |
+
for repeat in range(50):
|
| 488 |
+
rows.append(
|
| 489 |
+
_row(
|
| 490 |
+
session_start=session,
|
| 491 |
+
repeat=repeat,
|
| 492 |
+
case_id=f"case-{session}-{repeat}",
|
| 493 |
+
)
|
| 494 |
+
)
|
| 495 |
+
# warmup per session = 10, rows = 150 -> two thresholds satisfied,
|
| 496 |
+
# but session count < 5
|
| 497 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 498 |
+
summary = validate_and_aggregate(input_path)
|
| 499 |
+
comp = summary["measurement_completeness"]
|
| 500 |
+
assert comp["complete"] is False
|
| 501 |
+
assert comp["actual_session_starts"] == 3
|
| 502 |
+
reasons = " ".join(comp["incomplete_reasons"])
|
| 503 |
+
assert "independent session starts" in reasons
|
| 504 |
+
|
| 505 |
+
def test_insufficient_warmup_reported_incomplete(self, tmp_path: Path) -> None:
|
| 506 |
+
rows = _build_120row_fixture(warmup=5)
|
| 507 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 508 |
+
summary = validate_and_aggregate(input_path)
|
| 509 |
+
comp = summary["measurement_completeness"]
|
| 510 |
+
assert comp["complete"] is False
|
| 511 |
+
assert comp["actual_warmup_excluded_per_session"] == 5
|
| 512 |
+
|
| 513 |
+
def test_120rows_10sessions_is_complete(self, tmp_path: Path) -> None:
|
| 514 |
+
rows = _build_120row_fixture()
|
| 515 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 516 |
+
summary = validate_and_aggregate(input_path)
|
| 517 |
+
assert summary["measurement_completeness"]["complete"] is True
|
| 518 |
+
assert summary["measurement_completeness"]["incomplete_reasons"] == []
|
| 519 |
+
|
| 520 |
+
def test_page_semantics_not_process_cold_claimable(self, tmp_path: Path) -> None:
|
| 521 |
+
rows = [_row(session_start=1, repeat=0)]
|
| 522 |
+
rows[0]["provenance"]["session_start_semantics"] = "in_page_repeat"
|
| 523 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 524 |
+
summary = validate_and_aggregate(input_path)
|
| 525 |
+
assert summary["process_cold_claimable"] is False
|
| 526 |
+
|
| 527 |
+
def test_process_restart_semantics_still_false_v1(self, tmp_path: Path) -> None:
|
| 528 |
+
rows = [_row(session_start=1, repeat=0)]
|
| 529 |
+
rows[0]["provenance"]["session_start_semantics"] = "process_restart"
|
| 530 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 531 |
+
summary = validate_and_aggregate(input_path)
|
| 532 |
+
assert summary["process_cold_claimable"] is False
|
| 533 |
+
assert "always False regardless of semantics string" in summary["process_cold_note"]
|
| 534 |
+
|
| 535 |
+
def test_requested_provider_match_still_false_v1(self, tmp_path: Path) -> None:
|
| 536 |
+
rows = [
|
| 537 |
+
_row(session_start=1, repeat=0, observed_provider="wasm"),
|
| 538 |
+
_row(session_start=1, repeat=1, observed_provider="wasm"),
|
| 539 |
+
]
|
| 540 |
+
rows[0]["provenance"]["execution_provider_evidence"] = "window.ort.env.backend='wasm'"
|
| 541 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 542 |
+
summary = validate_and_aggregate(input_path)
|
| 543 |
+
assert summary["requested_is_proven"] is False
|
| 544 |
+
assert summary["experimental_condition"]["observed_provider"] == "wasm"
|
| 545 |
+
assert summary["experimental_condition"]["provenance"]["execution_provider_evidence"] == "window.ort.env.backend='wasm'"
|
| 546 |
+
assert "requested_is_proven is always False in v1" in summary["provider_claim_note"]
|
| 547 |
+
|
| 548 |
+
def test_observed_provider_mismatch_not_proven(self, tmp_path: Path) -> None:
|
| 549 |
+
rows = [_row(session_start=1, repeat=0, observed_provider=None)]
|
| 550 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 551 |
+
summary = validate_and_aggregate(input_path)
|
| 552 |
+
assert summary["requested_is_proven"] is False
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
# ---------------------------------------------------------------------------
|
| 556 |
+
# CLI integration
|
| 557 |
+
# ---------------------------------------------------------------------------
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
class TestCLI:
|
| 561 |
+
def test_cli_writes_summary(self, tmp_path: Path) -> None:
|
| 562 |
+
rows = _build_120row_fixture()
|
| 563 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 564 |
+
output_path = tmp_path / "summary.json"
|
| 565 |
+
result = subprocess.run(
|
| 566 |
+
[
|
| 567 |
+
sys.executable,
|
| 568 |
+
str(SUMMARY_TOOL),
|
| 569 |
+
"--input",
|
| 570 |
+
str(input_path),
|
| 571 |
+
"--output",
|
| 572 |
+
str(output_path),
|
| 573 |
+
],
|
| 574 |
+
capture_output=True,
|
| 575 |
+
check=False,
|
| 576 |
+
text=True,
|
| 577 |
+
)
|
| 578 |
+
assert result.returncode == 0, result.stderr
|
| 579 |
+
assert output_path.exists()
|
| 580 |
+
loaded = json.loads(output_path.read_text(encoding="utf-8"))
|
| 581 |
+
assert loaded["schema"] == "vons.browser-benchmark-summary/v1"
|
| 582 |
+
assert loaded["sample_counts"]["successful"] == 120
|
| 583 |
+
assert loaded["stage_percentiles_ms"]["totalRequestMs"]["p50"] == pytest.approx(60.5)
|
| 584 |
+
|
| 585 |
+
def test_cli_validation_fails_before_writing(self, tmp_path: Path) -> None:
|
| 586 |
+
rows = [
|
| 587 |
+
_row(session_start=1, repeat=0),
|
| 588 |
+
_row(session_start=1, repeat=0, case_id="dup"),
|
| 589 |
+
]
|
| 590 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 591 |
+
output_path = tmp_path / "summary.json"
|
| 592 |
+
result = subprocess.run(
|
| 593 |
+
[
|
| 594 |
+
sys.executable,
|
| 595 |
+
str(SUMMARY_TOOL),
|
| 596 |
+
"--input",
|
| 597 |
+
str(input_path),
|
| 598 |
+
"--output",
|
| 599 |
+
str(output_path),
|
| 600 |
+
],
|
| 601 |
+
capture_output=True,
|
| 602 |
+
check=False,
|
| 603 |
+
text=True,
|
| 604 |
+
)
|
| 605 |
+
assert result.returncode != 0
|
| 606 |
+
assert not output_path.exists() or output_path.stat().st_size == 0
|
| 607 |
+
|
| 608 |
+
|
| 609 |
+
# ---------------------------------------------------------------------------
|
| 610 |
+
# Error rows preserve nullable timing slots at summary level (not erased)
|
| 611 |
+
# ---------------------------------------------------------------------------
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
class TestErrorRowPreservation:
|
| 615 |
+
def test_timeout_rows_counted_as_failed_with_nullable_slots(self, tmp_path: Path) -> None:
|
| 616 |
+
rows = [
|
| 617 |
+
_row(session_start=1, repeat=0, status="timeout", error="gpu hang"),
|
| 618 |
+
_row(session_start=1, repeat=1, status="ok"),
|
| 619 |
+
]
|
| 620 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 621 |
+
summary = validate_and_aggregate(input_path)
|
| 622 |
+
assert summary["sample_counts"]["failed"] == 1
|
| 623 |
+
assert summary["sample_counts"]["successful"] == 1
|
| 624 |
+
# successful timings are only row[1]; all percentiles equal that row's values
|
| 625 |
+
assert summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] == pytest.approx(11.75)
|
| 626 |
+
|
| 627 |
+
|
| 628 |
+
# ---------------------------------------------------------------------------
|
| 629 |
+
# Integrator regressions: v1 hard-pessimism, seed 0, partial error timings,
|
| 630 |
+
# shape bounds
|
| 631 |
+
# ---------------------------------------------------------------------------
|
| 632 |
+
|
| 633 |
+
|
| 634 |
+
class TestIntegratorRegressions:
|
| 635 |
+
def test_unknown_semantics_still_false_process_cold(self, tmp_path: Path) -> None:
|
| 636 |
+
rows = [_row(session_start=1, repeat=0)]
|
| 637 |
+
rows[0]["provenance"]["session_start_semantics"] = "totally_unknown_semantic_XYZ"
|
| 638 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 639 |
+
summary = validate_and_aggregate(input_path)
|
| 640 |
+
assert summary["process_cold_claimable"] is False
|
| 641 |
+
assert summary["experimental_condition"]["provenance"]["session_start_semantics"] == "totally_unknown_semantic_XYZ"
|
| 642 |
+
assert "always False regardless of semantics string" in summary["process_cold_note"]
|
| 643 |
+
|
| 644 |
+
def test_matching_provider_unverified_evidence_not_proven(self, tmp_path: Path) -> None:
|
| 645 |
+
rows = [_row(session_start=1, repeat=0, observed_provider="wasm")]
|
| 646 |
+
rows[0]["requested_provider"] = "wasm"
|
| 647 |
+
rows[0]["observed_provider"] = "wasm"
|
| 648 |
+
rows[0]["provenance"]["execution_provider_evidence"] = "raw browser string: wasm observed"
|
| 649 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 650 |
+
summary = validate_and_aggregate(input_path)
|
| 651 |
+
assert summary["requested_is_proven"] is False
|
| 652 |
+
assert summary["experimental_condition"]["requested_provider"] == "wasm"
|
| 653 |
+
assert summary["experimental_condition"]["observed_provider"] == "wasm"
|
| 654 |
+
assert summary["experimental_condition"]["provenance"]["execution_provider_evidence"] == "raw browser string: wasm observed"
|
| 655 |
+
assert "no structured verified execution artifact" in summary["provider_claim_note"]
|
| 656 |
+
|
| 657 |
+
def test_seed_zero_is_valid(self, tmp_path: Path) -> None:
|
| 658 |
+
rows = _build_120row_fixture()
|
| 659 |
+
for row in rows:
|
| 660 |
+
row["seed"] = 0
|
| 661 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 662 |
+
summary = validate_and_aggregate(input_path)
|
| 663 |
+
assert summary["sample_counts"]["successful"] == 120
|
| 664 |
+
assert summary["experimental_condition"]["candidate_count"] == 4
|
| 665 |
+
p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"]
|
| 666 |
+
assert p50 == pytest.approx(60.5)
|
| 667 |
+
|
| 668 |
+
def test_partial_measured_error_row_validated_and_excluded_from_percentiles(self, tmp_path: Path) -> None:
|
| 669 |
+
rows = _build_120row_fixture()
|
| 670 |
+
error_row = _row(
|
| 671 |
+
session_start=11,
|
| 672 |
+
repeat=0,
|
| 673 |
+
case_id="case-extra-error",
|
| 674 |
+
status="error",
|
| 675 |
+
error="inference timeout after prepare",
|
| 676 |
+
timings={
|
| 677 |
+
"tokenizeMs": 12.0,
|
| 678 |
+
"prepareFeedMs": 5.5,
|
| 679 |
+
"totalRequestMs": 500.0,
|
| 680 |
+
},
|
| 681 |
+
)
|
| 682 |
+
rows.append(error_row)
|
| 683 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 684 |
+
summary = validate_and_aggregate(input_path)
|
| 685 |
+
assert summary["sample_counts"]["successful"] == 120
|
| 686 |
+
assert summary["sample_counts"]["failed"] == 1
|
| 687 |
+
p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"]
|
| 688 |
+
p95 = summary["stage_percentiles_ms"]["totalRequestMs"]["p95"]
|
| 689 |
+
assert p50 == pytest.approx(60.5)
|
| 690 |
+
assert p95 == pytest.approx(114.05)
|
| 691 |
+
|
| 692 |
+
def test_shape_candidate_exceeds_padded_rejected(self) -> None:
|
| 693 |
+
row = _row(candidate_count=10, padded_option_slots=8)
|
| 694 |
+
with pytest.raises(ValueError, match="candidate_count must be <= padded_option_slots"):
|
| 695 |
+
_validate_row(row, 1)
|
| 696 |
+
|
| 697 |
+
def test_shape_padded_exceeds_32_rejected(self) -> None:
|
| 698 |
+
row = _row(candidate_count=16, padded_option_slots=33)
|
| 699 |
+
with pytest.raises(ValueError, match="padded_option_slots must be <= 32"):
|
| 700 |
+
_validate_row(row, 1)
|
| 701 |
+
|
| 702 |
+
def test_shape_sequence_exceeds_512_rejected(self) -> None:
|
| 703 |
+
row = _row(sequence_length=513)
|
| 704 |
+
with pytest.raises(ValueError, match="sequence_length must be <= 512"):
|
| 705 |
+
_validate_row(row, 1)
|
| 706 |
+
|
| 707 |
+
def test_shape_boundary_values_accepted(self) -> None:
|
| 708 |
+
row = _row(candidate_count=32, padded_option_slots=32, sequence_length=512)
|
| 709 |
+
_validate_row(row, 1)
|
| 710 |
+
|
| 711 |
+
|
| 712 |
+
# ---------------------------------------------------------------------------
|
| 713 |
+
# --report-input mode: normalize live_candidates/allocated_candidates/
|
| 714 |
+
# sequence_tokens from existing v1 browser-benchmark JSON report, preserve
|
| 715 |
+
# top-level provenance, and never invent missing values
|
| 716 |
+
# ---------------------------------------------------------------------------
|
| 717 |
+
|
| 718 |
+
|
| 719 |
+
def _build_v1_report_fixture(
|
| 720 |
+
*,
|
| 721 |
+
backend: str = "direct",
|
| 722 |
+
live_field_name: str = "live_candidates",
|
| 723 |
+
allocated_field_name: str = "allocated_candidates",
|
| 724 |
+
seq_field_name: str = "sequence_length",
|
| 725 |
+
top_level_ort: str | None = "1.20.1",
|
| 726 |
+
include_session_start_semantics: bool = True,
|
| 727 |
+
include_environment: bool = True,
|
| 728 |
+
extra_failures: int = 0,
|
| 729 |
+
) -> dict[str, Any]:
|
| 730 |
+
samples: list[dict[str, Any]] = []
|
| 731 |
+
counter = 0
|
| 732 |
+
for session in range(1, 11):
|
| 733 |
+
for repeat in range(12):
|
| 734 |
+
counter += 1
|
| 735 |
+
if extra_failures > 0 and counter % 15 == 0:
|
| 736 |
+
sample: dict[str, Any] = {
|
| 737 |
+
"run_id": "run-alpha",
|
| 738 |
+
"session_start": session,
|
| 739 |
+
"repeat": repeat,
|
| 740 |
+
"case_id": f"case-{counter}",
|
| 741 |
+
"backend": backend,
|
| 742 |
+
"requested_provider": "wasm",
|
| 743 |
+
"observed_provider": "wasm",
|
| 744 |
+
"bundle_hash": SHA_A,
|
| 745 |
+
"tokenizer_hash": SHA_B,
|
| 746 |
+
"input_hash": SHA_C,
|
| 747 |
+
"noise_hash": None,
|
| 748 |
+
live_field_name: 4,
|
| 749 |
+
allocated_field_name: 8,
|
| 750 |
+
seq_field_name: 64,
|
| 751 |
+
"seed": 42,
|
| 752 |
+
"ort_version": top_level_ort,
|
| 753 |
+
"status": "error",
|
| 754 |
+
"error": "simulated",
|
| 755 |
+
"tokenizeMs": None,
|
| 756 |
+
"prepareFeedMs": None,
|
| 757 |
+
"inferenceAndReadbackMs": None,
|
| 758 |
+
"postprocessMs": None,
|
| 759 |
+
"totalRequestMs": None,
|
| 760 |
+
}
|
| 761 |
+
if backend == "diffusion":
|
| 762 |
+
sample["noise_hash"] = SHA_NOISE
|
| 763 |
+
samples.append(sample)
|
| 764 |
+
extra_failures -= 1
|
| 765 |
+
else:
|
| 766 |
+
total = float(counter)
|
| 767 |
+
tokenize = 0.1 * counter
|
| 768 |
+
prepare = 0.05 * counter
|
| 769 |
+
inference = 0.8 * counter
|
| 770 |
+
post = 0.05 * counter
|
| 771 |
+
sample = {
|
| 772 |
+
"run_id": "run-alpha",
|
| 773 |
+
"session_start": session,
|
| 774 |
+
"repeat": repeat,
|
| 775 |
+
"case_id": f"case-{counter}",
|
| 776 |
+
"backend": backend,
|
| 777 |
+
"requested_provider": "wasm",
|
| 778 |
+
"observed_provider": "wasm",
|
| 779 |
+
"bundle_hash": SHA_A,
|
| 780 |
+
"tokenizer_hash": SHA_B,
|
| 781 |
+
"input_hash": SHA_C,
|
| 782 |
+
"noise_hash": None,
|
| 783 |
+
live_field_name: 4,
|
| 784 |
+
allocated_field_name: 8,
|
| 785 |
+
seq_field_name: 64,
|
| 786 |
+
"seed": 42,
|
| 787 |
+
"ort_version": top_level_ort,
|
| 788 |
+
"status": "ok",
|
| 789 |
+
"error": None,
|
| 790 |
+
"tokenizeMs": tokenize,
|
| 791 |
+
"prepareFeedMs": prepare,
|
| 792 |
+
"inferenceAndReadbackMs": inference,
|
| 793 |
+
"postprocessMs": post,
|
| 794 |
+
"totalRequestMs": total,
|
| 795 |
+
"js_heap_used_bytes_before": 1000.0 + counter,
|
| 796 |
+
"js_heap_used_bytes_after": 2000.0 + counter,
|
| 797 |
+
}
|
| 798 |
+
if backend == "diffusion":
|
| 799 |
+
sample["noise_hash"] = SHA_NOISE
|
| 800 |
+
samples.append(sample)
|
| 801 |
+
report: dict[str, Any] = {
|
| 802 |
+
"schema": "vons.browser-benchmark/v1",
|
| 803 |
+
"run_id": "run-alpha",
|
| 804 |
+
"backend": backend,
|
| 805 |
+
"requested_provider": "wasm",
|
| 806 |
+
"warmup_excluded_per_session": 10,
|
| 807 |
+
"samples": samples,
|
| 808 |
+
}
|
| 809 |
+
if include_environment:
|
| 810 |
+
report["environment"] = dict(BASE_ENV)
|
| 811 |
+
report["execution_provider_evidence"] = "window.ort.env.backend='wasm'"
|
| 812 |
+
if include_session_start_semantics:
|
| 813 |
+
report["session_start_semantics"] = "page_reload"
|
| 814 |
+
if top_level_ort is not None:
|
| 815 |
+
report["ort_version"] = top_level_ort
|
| 816 |
+
return report
|
| 817 |
+
|
| 818 |
+
|
| 819 |
+
def _write_report(tmp_path: Path, report: dict[str, Any], *, name: str = "report.json") -> Path:
|
| 820 |
+
path = tmp_path / name
|
| 821 |
+
path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
|
| 822 |
+
return path
|
| 823 |
+
|
| 824 |
+
|
| 825 |
+
class TestReportInputNormalization:
|
| 826 |
+
def test_report_happy_path_matches_jsonl_percentiles(self, tmp_path: Path) -> None:
|
| 827 |
+
report = _build_v1_report_fixture()
|
| 828 |
+
report_path = _write_report(tmp_path, report)
|
| 829 |
+
summary = validate_and_aggregate_report(report_path)
|
| 830 |
+
assert summary["sample_counts"]["successful"] == 120
|
| 831 |
+
assert summary["sample_counts"]["failed"] == 0
|
| 832 |
+
assert summary["measurement_completeness"]["complete"] is True
|
| 833 |
+
p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"]
|
| 834 |
+
p95 = summary["stage_percentiles_ms"]["totalRequestMs"]["p95"]
|
| 835 |
+
assert p50 == pytest.approx(60.5)
|
| 836 |
+
assert p95 == pytest.approx(114.05)
|
| 837 |
+
assert summary["experimental_condition"]["provenance"]["session_start_semantics"] == "page_reload"
|
| 838 |
+
assert summary["process_cold_claimable"] is False
|
| 839 |
+
assert summary["requested_is_proven"] is False
|
| 840 |
+
|
| 841 |
+
def test_report_sequence_tokens_fallback_alias(self, tmp_path: Path) -> None:
|
| 842 |
+
report = _build_v1_report_fixture(seq_field_name="sequence_tokens")
|
| 843 |
+
for sample in report["samples"]:
|
| 844 |
+
assert "sequence_length" not in sample
|
| 845 |
+
assert sample["sequence_tokens"] == 64
|
| 846 |
+
report_path = _write_report(tmp_path, report)
|
| 847 |
+
summary = validate_and_aggregate_report(report_path)
|
| 848 |
+
assert summary["experimental_condition"]["sequence_length"] == 64
|
| 849 |
+
assert summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] == pytest.approx(60.5)
|
| 850 |
+
|
| 851 |
+
def test_report_missing_live_candidates_rejected(self, tmp_path: Path) -> None:
|
| 852 |
+
report = _build_v1_report_fixture()
|
| 853 |
+
for sample in report["samples"]:
|
| 854 |
+
del sample["live_candidates"]
|
| 855 |
+
report_path = _write_report(tmp_path, report)
|
| 856 |
+
with pytest.raises(ValueError, match="missing live_candidates"):
|
| 857 |
+
validate_and_aggregate_report(report_path)
|
| 858 |
+
|
| 859 |
+
def test_report_missing_allocated_candidates_rejected(self, tmp_path: Path) -> None:
|
| 860 |
+
report = _build_v1_report_fixture()
|
| 861 |
+
for sample in report["samples"]:
|
| 862 |
+
del sample["allocated_candidates"]
|
| 863 |
+
report_path = _write_report(tmp_path, report)
|
| 864 |
+
with pytest.raises(ValueError, match="missing allocated_candidates"):
|
| 865 |
+
validate_and_aggregate_report(report_path)
|
| 866 |
+
|
| 867 |
+
def test_report_missing_sequence_both_names_rejected(self, tmp_path: Path) -> None:
|
| 868 |
+
report = _build_v1_report_fixture()
|
| 869 |
+
for sample in report["samples"]:
|
| 870 |
+
del sample["sequence_length"]
|
| 871 |
+
report_path = _write_report(tmp_path, report)
|
| 872 |
+
with pytest.raises(ValueError, match="missing sequence_length"):
|
| 873 |
+
validate_and_aggregate_report(report_path)
|
| 874 |
+
|
| 875 |
+
def test_report_missing_session_start_semantics_rejected_no_invent(self, tmp_path: Path) -> None:
|
| 876 |
+
report = _build_v1_report_fixture(include_session_start_semantics=False)
|
| 877 |
+
assert "session_start_semantics" not in report
|
| 878 |
+
report_path = _write_report(tmp_path, report)
|
| 879 |
+
with pytest.raises(ValueError, match="do not invent missing values"):
|
| 880 |
+
validate_and_aggregate_report(report_path)
|
| 881 |
+
|
| 882 |
+
def test_report_missing_ort_version_sample_and_top_rejected(self, tmp_path: Path) -> None:
|
| 883 |
+
report = _build_v1_report_fixture(top_level_ort=None)
|
| 884 |
+
for sample in report["samples"]:
|
| 885 |
+
del sample["ort_version"]
|
| 886 |
+
report_path = _write_report(tmp_path, report)
|
| 887 |
+
with pytest.raises(ValueError, match="missing ort_version"):
|
| 888 |
+
validate_and_aggregate_report(report_path)
|
| 889 |
+
|
| 890 |
+
def test_report_top_level_ort_used_as_fallback(self, tmp_path: Path) -> None:
|
| 891 |
+
report = _build_v1_report_fixture(top_level_ort="1.21.0-custom")
|
| 892 |
+
for sample in report["samples"]:
|
| 893 |
+
del sample["ort_version"]
|
| 894 |
+
report_path = _write_report(tmp_path, report)
|
| 895 |
+
summary = validate_and_aggregate_report(report_path)
|
| 896 |
+
assert summary["experimental_condition"]["ort_version"] == "1.21.0-custom"
|
| 897 |
+
assert summary["sample_counts"]["successful"] == 120
|
| 898 |
+
|
| 899 |
+
def test_report_wrong_schema_rejected(self, tmp_path: Path) -> None:
|
| 900 |
+
report = _build_v1_report_fixture()
|
| 901 |
+
report["schema"] = "other.schema/v2"
|
| 902 |
+
report_path = _write_report(tmp_path, report)
|
| 903 |
+
with pytest.raises(ValueError, match="unexpected browser benchmark schema"):
|
| 904 |
+
validate_and_aggregate_report(report_path)
|
| 905 |
+
|
| 906 |
+
|
| 907 |
+
# ---------------------------------------------------------------------------
|
| 908 |
+
# CLI --report-input subprocess paths
|
| 909 |
+
# ---------------------------------------------------------------------------
|
| 910 |
+
|
| 911 |
+
|
| 912 |
+
class TestCLIReportInput:
|
| 913 |
+
def test_cli_report_input_writes_summary(self, tmp_path: Path) -> None:
|
| 914 |
+
report = _build_v1_report_fixture()
|
| 915 |
+
report_path = _write_report(tmp_path, report)
|
| 916 |
+
output_path = tmp_path / "summary.json"
|
| 917 |
+
result = subprocess.run(
|
| 918 |
+
[
|
| 919 |
+
sys.executable,
|
| 920 |
+
str(SUMMARY_TOOL),
|
| 921 |
+
"--report-input",
|
| 922 |
+
str(report_path),
|
| 923 |
+
"--output",
|
| 924 |
+
str(output_path),
|
| 925 |
+
],
|
| 926 |
+
capture_output=True,
|
| 927 |
+
check=False,
|
| 928 |
+
text=True,
|
| 929 |
+
)
|
| 930 |
+
assert result.returncode == 0, result.stderr
|
| 931 |
+
assert output_path.exists()
|
| 932 |
+
loaded = json.loads(output_path.read_text(encoding="utf-8"))
|
| 933 |
+
assert loaded["schema"] == "vons.browser-benchmark-summary/v1"
|
| 934 |
+
assert loaded["sample_counts"]["successful"] == 120
|
| 935 |
+
assert loaded["stage_percentiles_ms"]["totalRequestMs"]["p50"] == pytest.approx(60.5)
|
| 936 |
+
assert loaded["process_cold_claimable"] is False
|
| 937 |
+
assert loaded["requested_is_proven"] is False
|
| 938 |
+
|
| 939 |
+
def test_cli_mutually_exclusive_inputs(self, tmp_path: Path) -> None:
|
| 940 |
+
rows = _build_120row_fixture()
|
| 941 |
+
input_path = _write_jsonl(tmp_path, rows)
|
| 942 |
+
report = _build_v1_report_fixture()
|
| 943 |
+
report_path = _write_report(tmp_path, report)
|
| 944 |
+
output_path = tmp_path / "summary.json"
|
| 945 |
+
result = subprocess.run(
|
| 946 |
+
[
|
| 947 |
+
sys.executable,
|
| 948 |
+
str(SUMMARY_TOOL),
|
| 949 |
+
"--input",
|
| 950 |
+
str(input_path),
|
| 951 |
+
"--report-input",
|
| 952 |
+
str(report_path),
|
| 953 |
+
"--output",
|
| 954 |
+
str(output_path),
|
| 955 |
+
],
|
| 956 |
+
capture_output=True,
|
| 957 |
+
check=False,
|
| 958 |
+
text=True,
|
| 959 |
+
)
|
| 960 |
+
assert result.returncode != 0
|
| 961 |
+
assert "not allowed with" in result.stderr or "argument --report-input: not allowed" in result.stderr
|
| 962 |
+
|
| 963 |
+
|
| 964 |
+
# ---------------------------------------------------------------------------
|
| 965 |
+
# Pass 3: Real report compatibility regressions — matches actual report
|
| 966 |
+
# shape in reports/browser-direct-{wasm,webgpu-controlled}-merged.json:
|
| 967 |
+
# - ort_version via top-level runtime_info.ortVersion (camelCase, nested)
|
| 968 |
+
# - seed absent in samples => null allowed for direct backend when
|
| 969 |
+
# noise_hash=null (deterministic-direct not_applicable)
|
| 970 |
+
# - observed_provider=null (harness doesn't expose execution partition)
|
| 971 |
+
# - heap keys js_heap_used_bytes_before/after canonical
|
| 972 |
+
# ---------------------------------------------------------------------------
|
| 973 |
+
|
| 974 |
+
|
| 975 |
+
def _build_real_direct_report(
|
| 976 |
+
*,
|
| 977 |
+
runtime_ort: str = "1.30.0",
|
| 978 |
+
top_ort_version_key_present: bool = False,
|
| 979 |
+
top_ort_value: str | None = None,
|
| 980 |
+
seed_present_in_samples: bool = False,
|
| 981 |
+
noise_hash_value: None | str = None,
|
| 982 |
+
) -> dict[str, Any]:
|
| 983 |
+
samples: list[dict[str, Any]] = []
|
| 984 |
+
counter = 0
|
| 985 |
+
for session in range(1, 7):
|
| 986 |
+
for repeat in range(20):
|
| 987 |
+
counter += 1
|
| 988 |
+
total = float(counter)
|
| 989 |
+
sample: dict[str, Any] = {
|
| 990 |
+
"run_id": "run-real-shape",
|
| 991 |
+
"session_start": session,
|
| 992 |
+
"repeat": repeat,
|
| 993 |
+
"case_id": f"case-{counter}",
|
| 994 |
+
"backend": "direct",
|
| 995 |
+
"requested_provider": "wasm",
|
| 996 |
+
"observed_provider": None,
|
| 997 |
+
"observed_provider_status": "session_created; execution_partition_not_exposed",
|
| 998 |
+
"bundle_hash": SHA_A,
|
| 999 |
+
"tokenizer_hash": SHA_B,
|
| 1000 |
+
"input_hash": SHA_C,
|
| 1001 |
+
"noise_hash": noise_hash_value,
|
| 1002 |
+
"live_candidates": 4,
|
| 1003 |
+
"allocated_candidates": 8,
|
| 1004 |
+
"sequence_length": 64,
|
| 1005 |
+
"status": "ok",
|
| 1006 |
+
"error": None,
|
| 1007 |
+
"tokenizeMs": 0.1 * counter,
|
| 1008 |
+
"prepareFeedMs": 0.05 * counter,
|
| 1009 |
+
"inferenceAndReadbackMs": 0.8 * counter,
|
| 1010 |
+
"postprocessMs": 0.05 * counter,
|
| 1011 |
+
"totalRequestMs": total,
|
| 1012 |
+
"js_heap_used_bytes_before": 1000.0 + counter,
|
| 1013 |
+
"js_heap_used_bytes_after": 2000.0 + counter,
|
| 1014 |
+
}
|
| 1015 |
+
if seed_present_in_samples:
|
| 1016 |
+
sample["seed"] = 42
|
| 1017 |
+
samples.append(sample)
|
| 1018 |
+
report: dict[str, Any] = {
|
| 1019 |
+
"schema": "vons.browser-benchmark/v1",
|
| 1020 |
+
"run_id": "run-real-shape",
|
| 1021 |
+
"backend": "direct",
|
| 1022 |
+
"requested_provider": "wasm",
|
| 1023 |
+
"environment": dict(BASE_ENV),
|
| 1024 |
+
"execution_provider_evidence": "requested provider and session creation only",
|
| 1025 |
+
"session_start_semantics": "one fresh page/session per source report; browser caches may persist",
|
| 1026 |
+
"warmup_excluded_per_session": 10,
|
| 1027 |
+
"runtime_info": {
|
| 1028 |
+
"ortVersion": runtime_ort,
|
| 1029 |
+
"provider": "wasm",
|
| 1030 |
+
"wasmNumThreads": 1,
|
| 1031 |
+
"webgpuAdapterRequested": False,
|
| 1032 |
+
"webgpuDeviceRequested": False,
|
| 1033 |
+
},
|
| 1034 |
+
"samples": samples,
|
| 1035 |
+
}
|
| 1036 |
+
if top_ort_version_key_present:
|
| 1037 |
+
report["ort_version"] = top_ort_value
|
| 1038 |
+
return report
|
| 1039 |
+
|
| 1040 |
+
|
| 1041 |
+
class TestRealReportCompatibility:
|
| 1042 |
+
def test_runtime_info_ortversion_used_as_top_fallback(self, tmp_path: Path) -> None:
|
| 1043 |
+
report = _build_real_direct_report(runtime_ort="1.30.0-fromRuntimeInfo")
|
| 1044 |
+
report_path = _write_report(tmp_path, report)
|
| 1045 |
+
summary = validate_and_aggregate_report(report_path)
|
| 1046 |
+
assert summary["experimental_condition"]["ort_version"] == "1.30.0-fromRuntimeInfo"
|
| 1047 |
+
assert summary["sample_counts"]["successful"] == 120
|
| 1048 |
+
p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"]
|
| 1049 |
+
assert p50 == pytest.approx(60.5)
|
| 1050 |
+
|
| 1051 |
+
def test_top_ort_version_preferred_over_runtime_info(self, tmp_path: Path) -> None:
|
| 1052 |
+
report = _build_real_direct_report(
|
| 1053 |
+
runtime_ort="1.30.0-runtime-should-not-be-used",
|
| 1054 |
+
top_ort_version_key_present=True,
|
| 1055 |
+
top_ort_value="1.31.0-topPreferred",
|
| 1056 |
+
)
|
| 1057 |
+
report_path = _write_report(tmp_path, report)
|
| 1058 |
+
summary = validate_and_aggregate_report(report_path)
|
| 1059 |
+
assert summary["experimental_condition"]["ort_version"] == "1.31.0-topPreferred"
|
| 1060 |
+
|
| 1061 |
+
def test_missing_both_ort_raises_mentions_runtimeinfo(self, tmp_path: Path) -> None:
|
| 1062 |
+
report = _build_real_direct_report(runtime_ort="1.30.0")
|
| 1063 |
+
del report["runtime_info"]
|
| 1064 |
+
report_path = _write_report(tmp_path, report)
|
| 1065 |
+
with pytest.raises(ValueError, match="runtime_info.ortVersion"):
|
| 1066 |
+
validate_and_aggregate_report(report_path)
|
| 1067 |
+
|
| 1068 |
+
def test_direct_samples_no_seed_noisehash_null_resolves_seed_null(self, tmp_path: Path) -> None:
|
| 1069 |
+
report = _build_real_direct_report(seed_present_in_samples=False, noise_hash_value=None)
|
| 1070 |
+
for sample in report["samples"]:
|
| 1071 |
+
assert "seed" not in sample
|
| 1072 |
+
assert sample["noise_hash"] is None
|
| 1073 |
+
report_path = _write_report(tmp_path, report)
|
| 1074 |
+
summary = validate_and_aggregate_report(report_path)
|
| 1075 |
+
assert summary["experimental_condition"]["seed"] is None
|
| 1076 |
+
assert summary["sample_counts"]["successful"] == 120
|
| 1077 |
+
p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"]
|
| 1078 |
+
assert p50 == pytest.approx(60.5)
|
| 1079 |
+
|
| 1080 |
+
def test_direct_with_noise_hash_still_requires_seed_not_null(self, tmp_path: Path) -> None:
|
| 1081 |
+
report = _build_real_direct_report(seed_present_in_samples=False, noise_hash_value=SHA_NOISE)
|
| 1082 |
+
for sample in report["samples"]:
|
| 1083 |
+
assert "seed" not in sample
|
| 1084 |
+
assert sample["noise_hash"] == SHA_NOISE
|
| 1085 |
+
report_path = _write_report(tmp_path, report)
|
| 1086 |
+
with pytest.raises(ValueError, match="missing seed"):
|
| 1087 |
+
validate_and_aggregate_report(report_path)
|
| 1088 |
+
|
| 1089 |
+
def test_diffusion_samples_require_seed_never_null(self) -> None:
|
| 1090 |
+
row = _row(backend="diffusion", noise_hash=SHA_NOISE)
|
| 1091 |
+
row["seed"] = None
|
| 1092 |
+
with pytest.raises(ValueError, match="seed may only be null for direct backend"):
|
| 1093 |
+
_validate_row(row, 1)
|
| 1094 |
+
|
| 1095 |
+
def test_direct_with_seed_integer_still_accepted(self) -> None:
|
| 1096 |
+
row = _row(seed=17)
|
| 1097 |
+
row["noise_hash"] = None
|
| 1098 |
+
row["seed"] = 17
|
| 1099 |
+
_validate_row(row, 1)
|
| 1100 |
+
|
| 1101 |
+
def test_direct_seed_null_noise_hash_null_accepted_at_row_level(self) -> None:
|
| 1102 |
+
row = _row(seed=42)
|
| 1103 |
+
row["seed"] = None
|
| 1104 |
+
row["noise_hash"] = None
|
| 1105 |
+
_validate_row(row, 1)
|
| 1106 |
+
|
| 1107 |
+
def test_observed_provider_null_preserved_in_summary(self, tmp_path: Path) -> None:
|
| 1108 |
+
report = _build_real_direct_report()
|
| 1109 |
+
report_path = _write_report(tmp_path, report)
|
| 1110 |
+
summary = validate_and_aggregate_report(report_path)
|
| 1111 |
+
assert summary["experimental_condition"]["observed_provider"] is None
|
| 1112 |
+
assert summary["requested_is_proven"] is False
|
| 1113 |
+
|
| 1114 |
+
def test_heap_keys_canonical_and_values_match(self, tmp_path: Path) -> None:
|
| 1115 |
+
report = _build_real_direct_report()
|
| 1116 |
+
report_path = _write_report(tmp_path, report)
|
| 1117 |
+
summary = validate_and_aggregate_report(report_path)
|
| 1118 |
+
keys = sorted(summary["heap_maxima_bytes"].keys())
|
| 1119 |
+
assert keys == ["js_heap_used_bytes_after", "js_heap_used_bytes_before"]
|
| 1120 |
+
# counter = 1..120 => heap_before = 1000 + counter => max=1120
|
| 1121 |
+
assert summary["heap_maxima_bytes"]["js_heap_used_bytes_before"] == pytest.approx(1120.0)
|
| 1122 |
+
assert summary["heap_maxima_bytes"]["js_heap_used_bytes_after"] == pytest.approx(2120.0)
|
| 1123 |
+
|
| 1124 |
+
def test_completeness_passes_with_real_reports_120(self, tmp_path: Path) -> None:
|
| 1125 |
+
report = _build_real_direct_report()
|
| 1126 |
+
report_path = _write_report(tmp_path, report)
|
| 1127 |
+
summary = validate_and_aggregate_report(report_path)
|
| 1128 |
+
assert summary["measurement_completeness"]["complete"] is True
|
| 1129 |
+
assert summary["measurement_completeness"]["incomplete_reasons"] == []
|
| 1130 |
+
assert summary["sample_counts"]["unique_session_starts"] == 6
|
tests/test_browser_parity_reference.py
ADDED
|
@@ -0,0 +1,350 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Verifier regressions with tiny tokenizers and mocked CPU outputs only."""
|
| 2 |
+
|
| 3 |
+
import copy
|
| 4 |
+
import hashlib
|
| 5 |
+
import importlib.util
|
| 6 |
+
import json
|
| 7 |
+
import math
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from types import SimpleNamespace
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
import pytest
|
| 13 |
+
|
| 14 |
+
SPEC = importlib.util.spec_from_file_location(
|
| 15 |
+
"browser_parity_reference", Path(__file__).parents[1] / "tools/verify_browser_parity.py"
|
| 16 |
+
)
|
| 17 |
+
assert SPEC is not None and SPEC.loader is not None
|
| 18 |
+
verifier = importlib.util.module_from_spec(SPEC)
|
| 19 |
+
SPEC.loader.exec_module(verifier)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class TinyTokenizer:
|
| 23 |
+
def encode(self, text):
|
| 24 |
+
ids = [101, *text.encode("utf-8"), 102]
|
| 25 |
+
return SimpleNamespace(ids=ids, attention_mask=[1] * len(ids), type_ids=[0] * len(ids))
|
| 26 |
+
|
| 27 |
+
def token_to_id(self, token):
|
| 28 |
+
assert token == "[PAD]"
|
| 29 |
+
return 0
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class Reference:
|
| 33 |
+
def __init__(self):
|
| 34 |
+
self.feeds = []
|
| 35 |
+
self.answerability = 2.0
|
| 36 |
+
self.invalid = False
|
| 37 |
+
|
| 38 |
+
def run(self, feed, backend):
|
| 39 |
+
self.feeds.append(copy.deepcopy(feed))
|
| 40 |
+
scores = np.zeros(feed["option_mask"].shape, np.float32)
|
| 41 |
+
scores[0, 0] = float("nan") if self.invalid else 3.0
|
| 42 |
+
return scores, np.array([self.answerability], np.float32)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def question(key="q", kind="choice"):
|
| 46 |
+
return {"id": key, "type": kind, "prompt": "Choose", "options": ["first", "second"]}
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def expected_answer(q, logit=2.0):
|
| 50 |
+
options = q["options"] or ["true", "false"]
|
| 51 |
+
probability = 1 / (1 + math.exp(-3))
|
| 52 |
+
answerability = 1 / (1 + math.exp(-logit))
|
| 53 |
+
confidence = probability * answerability
|
| 54 |
+
reason = "answerability_below_threshold" if answerability < 0.5 else (
|
| 55 |
+
"confidence_below_threshold" if confidence < 0.55 else None
|
| 56 |
+
)
|
| 57 |
+
return {"question_id": q["id"], "choice": None if reason else options[0],
|
| 58 |
+
"probabilities": [{"option": options[0], "probability": probability},
|
| 59 |
+
{"option": options[1], "probability": 1 - probability}],
|
| 60 |
+
"confidence": confidence, "status": "abstain" if reason else "ok",
|
| 61 |
+
"abstain_reason": reason}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def fixture(backend="direct", multi=False):
|
| 65 |
+
questions = [question()]
|
| 66 |
+
if multi:
|
| 67 |
+
questions.append({"id": "q2", "type": "boolean", "prompt": "Continue?", "options": []})
|
| 68 |
+
request = {"state": "ready", "questions": questions, "seed": 7}
|
| 69 |
+
metadata = {"backend": backend, "sequence_length": 128, "option_count": 4}
|
| 70 |
+
manifest = {"metadata": metadata}
|
| 71 |
+
noise = [0.25, -0.5, 1.0, 2.0] if backend == "diffusion" else None
|
| 72 |
+
traces, timings, answers = [], [], []
|
| 73 |
+
for q in questions:
|
| 74 |
+
options = q["options"] or ["true", "false"]
|
| 75 |
+
slots = 4 if backend == "diffusion" else 2
|
| 76 |
+
ids, masks, types, live = [], [], [], 0
|
| 77 |
+
for index in range(slots):
|
| 78 |
+
raw = [101, *f"ready\nQuestion: {q['prompt']}\nCandidate: {options[index]}".encode(), 102] \
|
| 79 |
+
if index < 2 else []
|
| 80 |
+
live += len(raw)
|
| 81 |
+
ids.extend(raw + [0] * (128 - len(raw)))
|
| 82 |
+
masks.extend([1] * len(raw) + [0] * (128 - len(raw)))
|
| 83 |
+
types.extend([0] * 128)
|
| 84 |
+
answer = expected_answer(q)
|
| 85 |
+
traces.append({"questionId": q["id"], "inputIds": ids, "attentionMask": masks,
|
| 86 |
+
"tokenTypeIds": types, "optionMask": [1, 1] + [0] * (slots - 2),
|
| 87 |
+
"initialNoise": noise, "rawScores": [3.0, 0.0], "answerabilityLogit": 2.0,
|
| 88 |
+
"probabilities": [p["probability"] for p in answer["probabilities"]],
|
| 89 |
+
"answer": copy.deepcopy(answer)})
|
| 90 |
+
timings.append({"questionId": q["id"], "live_candidates": 2, "allocated_candidates": slots,
|
| 91 |
+
"live_tokens": live, "sequence_length": 128})
|
| 92 |
+
answers.append(answer)
|
| 93 |
+
row = {"fixture_id": "fixture", "input_hash": verifier.request_hash(request),
|
| 94 |
+
"request": request, "status": "ok", "traces": traces, "timings": timings,
|
| 95 |
+
"response": {"backend": backend, "model_id": f"vons-{backend}-onnx-web",
|
| 96 |
+
"answers": answers}}
|
| 97 |
+
report = {"schema": verifier.SCHEMA, "backend": backend, "fixture_count": 1,
|
| 98 |
+
"noise_values": noise, "records": [row]}
|
| 99 |
+
return report, manifest
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def run(report, manifest, reference=None):
|
| 103 |
+
return verifier.verify_records(report, manifest, TinyTokenizer(), reference or Reference())
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def test_canonical_hash_matches_browser_fixture_domain_and_unicode_values():
|
| 107 |
+
value = {"state": {"z": "🌐 검토", "a": 2}, "seed": 7, "questions": []}
|
| 108 |
+
canonical = '{"questions":[],"seed":7,"state":{"a":2,"z":"🌐 검토"}}'
|
| 109 |
+
assert verifier.stable_json(value) == canonical
|
| 110 |
+
assert verifier.request_hash(value) == hashlib.sha256(canonical.encode()).hexdigest()
|
| 111 |
+
assert verifier.stable_json({"seed": -0.0}) == '{"seed":0}'
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
@pytest.mark.parametrize("value", [{"Mixed": 1}, {"a_b": 1}, {"é": 1}, {"a": 1.25},
|
| 115 |
+
{"a": 2**53}, {"a": float("nan")}])
|
| 116 |
+
def test_unsupported_json_canonicalization_fails_closed(value):
|
| 117 |
+
with pytest.raises(verifier.ParityError):
|
| 118 |
+
verifier.stable_json(value)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
@pytest.mark.parametrize("backend", ["direct", "diffusion"])
|
| 122 |
+
def test_every_multiquestion_trace_and_response_compares(backend):
|
| 123 |
+
report, manifest = fixture(backend, multi=True)
|
| 124 |
+
reference = Reference()
|
| 125 |
+
result = run(report, manifest, reference)
|
| 126 |
+
assert result["counts"] == {"passed": 1, "failed": 0, "unverified": 0,
|
| 127 |
+
"contract_error_verified": 0}
|
| 128 |
+
assert result["numerical_questions"] == {"passed": 2, "failed": 0}
|
| 129 |
+
assert result["all_numerical_comparisons_passed"]
|
| 130 |
+
assert len(reference.feeds) == 2
|
| 131 |
+
assert result["calibration"] is None
|
| 132 |
+
if backend == "diffusion":
|
| 133 |
+
expected = hashlib.sha256(np.array(report["noise_values"], dtype="<f4").tobytes()).hexdigest()
|
| 134 |
+
assert all(q["noise_float32_le_sha256"] == expected for q in result["records"][0]["questions"])
|
| 135 |
+
np.testing.assert_array_equal(reference.feeds[0]["initial_noise"], [[0.25, -0.5, 1, 2]])
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
@pytest.mark.parametrize("field", ["inputIds", "attentionMask", "tokenTypeIds", "optionMask"])
|
| 139 |
+
def test_exact_token_arrays_are_required_before_inference(field):
|
| 140 |
+
report, manifest = fixture()
|
| 141 |
+
report["records"][0]["traces"][0][field][0] += 1
|
| 142 |
+
reference = Reference()
|
| 143 |
+
result = run(report, manifest, reference)
|
| 144 |
+
assert result["counts"]["failed"] == 1
|
| 145 |
+
assert not reference.feeds
|
| 146 |
+
assert field in result["records"][0]["questions"][0]["reason"]
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
@pytest.mark.parametrize("mutation", ["missing_trace", "missing_hash", "bad_hash", "missing_question",
|
| 150 |
+
"wrong_shape", "response_choice", "response_probability"])
|
| 151 |
+
def test_missing_or_tampered_evidence_cannot_pass(mutation):
|
| 152 |
+
report, manifest = fixture(multi=True)
|
| 153 |
+
row = report["records"][0]
|
| 154 |
+
if mutation == "missing_trace":
|
| 155 |
+
row["traces"] = []
|
| 156 |
+
elif mutation == "missing_hash":
|
| 157 |
+
del row["input_hash"]
|
| 158 |
+
elif mutation == "bad_hash":
|
| 159 |
+
row["input_hash"] = "0" * 64
|
| 160 |
+
elif mutation == "missing_question":
|
| 161 |
+
row["response"]["answers"].pop()
|
| 162 |
+
elif mutation == "wrong_shape":
|
| 163 |
+
row["timings"][0]["sequence_length"] = 512
|
| 164 |
+
elif mutation == "response_choice":
|
| 165 |
+
row["response"]["answers"][1]["choice"] = "false"
|
| 166 |
+
else:
|
| 167 |
+
row["response"]["answers"][1]["probabilities"][0]["probability"] = 0.1
|
| 168 |
+
result = run(report, manifest)
|
| 169 |
+
assert result["counts"]["failed"] == 1
|
| 170 |
+
assert not result["all_numerical_comparisons_passed"]
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
@pytest.mark.parametrize("field", ["rawScores", "probabilities", "answerabilityLogit"])
|
| 174 |
+
def test_nonfinite_browser_values_never_pass(field):
|
| 175 |
+
report, manifest = fixture()
|
| 176 |
+
trace = report["records"][0]["traces"][0]
|
| 177 |
+
trace[field] = float("nan") if field == "answerabilityLogit" else [float("nan"), 0.0]
|
| 178 |
+
result = run(report, manifest)
|
| 179 |
+
assert result["counts"]["failed"] == 1
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def test_nonfinite_cpu_values_never_pass_even_with_large_tolerance():
|
| 183 |
+
report, manifest = fixture()
|
| 184 |
+
reference = Reference()
|
| 185 |
+
reference.invalid = True
|
| 186 |
+
result = verifier.verify_records(report, manifest, TinyTokenizer(), reference, atol=100, rtol=100)
|
| 187 |
+
assert result["counts"]["failed"] == 1
|
| 188 |
+
assert "non-finite" in result["records"][0]["questions"][0]["reason"]
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def test_noise_uses_array_identity_not_seed_identity():
|
| 192 |
+
report, manifest = fixture("diffusion")
|
| 193 |
+
report["records"][0]["traces"][0]["initialNoise"] = [0.5, -0.5, 1.0, 2.0]
|
| 194 |
+
reference = Reference()
|
| 195 |
+
result = run(report, manifest, reference)
|
| 196 |
+
assert result["counts"]["failed"] == 1
|
| 197 |
+
assert not reference.feeds
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def test_non_float32_noise_fails_before_inference():
|
| 201 |
+
report, manifest = fixture("diffusion")
|
| 202 |
+
report["noise_values"][0] = 0.1
|
| 203 |
+
result = run(report, manifest)
|
| 204 |
+
assert "exact Float32" in result["records"][0]["questions"][0]["reason"]
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def test_postprocess_both_abstention_gates_and_tie_order():
|
| 208 |
+
answer = verifier.postprocess([4.0, 0.0], -1.0, ["first", "second"], "q")
|
| 209 |
+
assert answer["abstain_reason"] == "answerability_below_threshold" and answer["choice"] is None
|
| 210 |
+
answer = verifier.postprocess([0.0, 0.0], 20.0, ["first", "second"], "q")
|
| 211 |
+
assert answer["abstain_reason"] == "confidence_below_threshold" and answer["choice"] is None
|
| 212 |
+
answer = verifier.postprocess([3.0, 0.0], 2.0, ["first", "second"], "q")
|
| 213 |
+
assert answer["choice"] == "first" and answer["status"] == "ok"
|
| 214 |
+
with pytest.raises(verifier.ParityError):
|
| 215 |
+
verifier.postprocess([-math.inf, -math.inf], 2, ["first", "second"], "q")
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def test_score_unsupported_is_unverified_not_contract_or_numerical_pass():
|
| 219 |
+
report, manifest = fixture()
|
| 220 |
+
row = report["records"][0]
|
| 221 |
+
row["request"]["questions"][0].update(type="score", rubric=["poor", "good"])
|
| 222 |
+
row["input_hash"] = verifier.request_hash(row["request"])
|
| 223 |
+
row.update(status="expected_error", traces=[], timings=[],
|
| 224 |
+
error="Error: score questions are not supported by this candidate-selection ONNX head")
|
| 225 |
+
result = run(report, manifest)
|
| 226 |
+
assert result["counts"]["unverified"] == 1
|
| 227 |
+
assert result["numerical_questions"] == {"passed": 0, "failed": 0}
|
| 228 |
+
assert not result["all_numerical_comparisons_passed"]
|
| 229 |
+
assert not result["complete_contract_verified"]
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def test_expected_overflow_requires_independent_exact_tokenization():
|
| 233 |
+
report, manifest = fixture()
|
| 234 |
+
row = report["records"][0]
|
| 235 |
+
row.update(status="expected_error", traces=[], timings=[], error="RangeError: token overflow")
|
| 236 |
+
assert run(report, manifest)["counts"]["failed"] == 1
|
| 237 |
+
row["request"]["state"] = "x" * 200
|
| 238 |
+
row["input_hash"] = verifier.request_hash(row["request"])
|
| 239 |
+
result = run(report, manifest)
|
| 240 |
+
assert result["counts"]["contract_error_verified"] == 1
|
| 241 |
+
assert result["numerical_questions"]["passed"] == 0
|
| 242 |
+
assert not result["all_numerical_comparisons_passed"]
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def test_aggregate_budget_can_overflow_when_individual_candidates_fit():
|
| 246 |
+
q = question()
|
| 247 |
+
q["options"] = ["x" * 60, "y" * 60]
|
| 248 |
+
with pytest.raises(verifier.InputOverflow, match="aggregate"):
|
| 249 |
+
verifier.build_inputs(TinyTokenizer(), "ready", q,
|
| 250 |
+
{"sequence_length": 128, "option_count": 32}, "direct")
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def test_empty_duplicate_and_inconsistent_fixture_sets_fail():
|
| 254 |
+
report, manifest = fixture()
|
| 255 |
+
duplicate = copy.deepcopy(report["records"][0])
|
| 256 |
+
report["records"].append(duplicate)
|
| 257 |
+
report["fixture_count"] = 2
|
| 258 |
+
assert run(report, manifest)["counts"]["failed"] == 1
|
| 259 |
+
report["fixture_count"] = 3
|
| 260 |
+
with pytest.raises(verifier.ParityError, match="fixture_count"):
|
| 261 |
+
run(report, manifest)
|
| 262 |
+
report.update(records=[], fixture_count=0)
|
| 263 |
+
with pytest.raises(verifier.ParityError, match="nonempty"):
|
| 264 |
+
run(report, manifest)
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def make_artifacts(tmp_path):
|
| 268 |
+
root = tmp_path / "bundle"
|
| 269 |
+
root.mkdir()
|
| 270 |
+
graph, tokenizer = root / "model.onnx", root / "tokenizer.json"
|
| 271 |
+
graph.write_bytes(b"mock graph; no ORT load")
|
| 272 |
+
tokenizer.write_text("{}")
|
| 273 |
+
records = [{"path": path.name, "bytes": path.stat().st_size,
|
| 274 |
+
"sha256": verifier.file_hash(path), "role": role}
|
| 275 |
+
for path, role in [(graph, "model_graph"), (tokenizer, "tokenizer")]]
|
| 276 |
+
manifest_path = root / "bundle-manifest-v1.json"
|
| 277 |
+
manifest_path.write_text(json.dumps({"schema_version": "vons.bundle.manifest/v1",
|
| 278 |
+
"metadata": {"backend": "direct"}, "files": records}))
|
| 279 |
+
report = {"schema": verifier.SCHEMA, "backend": "direct",
|
| 280 |
+
"requested_provider": "wasm",
|
| 281 |
+
"manifest_hash": verifier.file_hash(manifest_path),
|
| 282 |
+
"tokenizer_hash": verifier.file_hash(tokenizer)}
|
| 283 |
+
return root, report, manifest_path
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def test_artifact_identity_pinned_before_reference_loading(tmp_path, monkeypatch):
|
| 287 |
+
root, report, manifest_path = make_artifacts(tmp_path)
|
| 288 |
+
calls = []
|
| 289 |
+
monkeypatch.setattr(verifier, "verify_bundle_manifest", lambda path: calls.append(path) or {"pass": True})
|
| 290 |
+
artifacts = verifier.validate_artifacts(report, manifest_path)
|
| 291 |
+
assert calls == [manifest_path]
|
| 292 |
+
assert artifacts["graph"] == root / "model.onnx"
|
| 293 |
+
report["manifest_hash"] = "0" * 64
|
| 294 |
+
with pytest.raises(verifier.ParityError, match="manifest_hash"):
|
| 295 |
+
verifier.validate_artifacts(report, manifest_path)
|
| 296 |
+
assert calls == [manifest_path]
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
@pytest.mark.parametrize("mutation", ["missing_hash", "tokenizer_hash", "mutated_file", "missing_file"])
|
| 300 |
+
def test_missing_and_changed_artifacts_fail(tmp_path, monkeypatch, mutation):
|
| 301 |
+
root, report, path = make_artifacts(tmp_path)
|
| 302 |
+
monkeypatch.setattr(verifier, "verify_bundle_manifest", lambda path: {"pass": True})
|
| 303 |
+
if mutation == "missing_hash":
|
| 304 |
+
del report["manifest_hash"]
|
| 305 |
+
elif mutation == "tokenizer_hash":
|
| 306 |
+
report["tokenizer_hash"] = "0" * 64
|
| 307 |
+
elif mutation == "mutated_file":
|
| 308 |
+
(root / "model.onnx").write_bytes(b"tampered")
|
| 309 |
+
else:
|
| 310 |
+
(root / "model.onnx").unlink()
|
| 311 |
+
with pytest.raises(verifier.ParityError):
|
| 312 |
+
verifier.validate_artifacts(report, path)
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
@pytest.mark.parametrize("relative", ["../outside", "/absolute", "C:/drive", "https:remote", "x\\y"])
|
| 316 |
+
def test_manifest_path_escape_rejected(tmp_path, relative):
|
| 317 |
+
with pytest.raises(verifier.ParityError):
|
| 318 |
+
verifier._relative_file(tmp_path, relative)
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def test_symlink_parent_even_inside_bundle_rejected(tmp_path):
|
| 322 |
+
real = tmp_path / "real"
|
| 323 |
+
real.mkdir()
|
| 324 |
+
(real / "data").write_bytes(b"data")
|
| 325 |
+
(tmp_path / "linked").symlink_to(real, target_is_directory=True)
|
| 326 |
+
with pytest.raises(verifier.ParityError, match="symlink"):
|
| 327 |
+
verifier._relative_file(tmp_path, "linked/data")
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def test_json_duplicate_keys_and_nan_rejected(tmp_path):
|
| 331 |
+
path = tmp_path / "invalid.json"
|
| 332 |
+
for payload in ('{"x":1,"x":2}', '{"x":NaN}'):
|
| 333 |
+
path.write_text(payload)
|
| 334 |
+
with pytest.raises(verifier.ParityError):
|
| 335 |
+
verifier.read_json(path)
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
def test_existing_evidence_is_not_overwritten_or_loaded(tmp_path):
|
| 339 |
+
output = tmp_path / "evidence.json"
|
| 340 |
+
output.write_text("preserved")
|
| 341 |
+
with pytest.raises(FileExistsError, match="overwrite"):
|
| 342 |
+
verifier.verify_report(tmp_path / "missing-report", tmp_path / "missing-manifest", output)
|
| 343 |
+
assert output.read_text() == "preserved"
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
@pytest.mark.parametrize("atol,rtol", [(math.nan, 1e-5), (1e-5, math.inf), (-1, 0)])
|
| 347 |
+
def test_invalid_tolerances_rejected(atol, rtol):
|
| 348 |
+
report, manifest = fixture()
|
| 349 |
+
with pytest.raises(verifier.ParityError, match="tolerances"):
|
| 350 |
+
verifier.verify_records(report, manifest, TinyTokenizer(), Reference(), atol, rtol)
|
tests/test_bundle.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import pytest
|
| 7 |
+
|
| 8 |
+
from vons.bundle import (
|
| 9 |
+
BundleManifestError,
|
| 10 |
+
_snapshot_hash,
|
| 11 |
+
build_bundle_manifest,
|
| 12 |
+
verify_bundle_manifest,
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _write_external_model(root: Path) -> tuple[Path, Path]:
|
| 17 |
+
onnx = pytest.importorskip("onnx")
|
| 18 |
+
from onnx import TensorProto, helper, numpy_helper
|
| 19 |
+
|
| 20 |
+
root.mkdir(parents=True, exist_ok=True)
|
| 21 |
+
initializer = numpy_helper.from_array(__import__("numpy").ones(2048, dtype="float32"), name="weight")
|
| 22 |
+
graph = helper.make_graph(
|
| 23 |
+
[helper.make_node("Identity", ["input"], ["output"])],
|
| 24 |
+
"test",
|
| 25 |
+
[helper.make_tensor_value_info("input", TensorProto.FLOAT, [2])],
|
| 26 |
+
[helper.make_tensor_value_info("output", TensorProto.FLOAT, [2])],
|
| 27 |
+
[initializer],
|
| 28 |
+
)
|
| 29 |
+
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 18)])
|
| 30 |
+
graph_path = root / "model.onnx"
|
| 31 |
+
onnx.save_model(model, graph_path, save_as_external_data=True, all_tensors_to_one_file=True, location="weights.onnx.data")
|
| 32 |
+
tokenizer = root / "tokenizer"
|
| 33 |
+
tokenizer.mkdir()
|
| 34 |
+
(tokenizer / "tokenizer.json").write_text("{}\n", encoding="utf-8")
|
| 35 |
+
return graph_path, tokenizer
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def test_manifest_inventories_external_data_and_verifies_bytes(tmp_path: Path) -> None:
|
| 39 |
+
graph, tokenizer = _write_external_model(tmp_path)
|
| 40 |
+
output = tmp_path / "bundle-manifest.v1.json"
|
| 41 |
+
build_bundle_manifest(graph, tokenizer, output)
|
| 42 |
+
result = verify_bundle_manifest(output)
|
| 43 |
+
payload = json.loads(output.read_text(encoding="utf-8"))
|
| 44 |
+
|
| 45 |
+
assert result["pass"] is True
|
| 46 |
+
assert result["external_data_files"] == 1
|
| 47 |
+
assert {item["role"] for item in payload["files"]} >= {"model_graph", "model_external_data", "tokenizer"}
|
| 48 |
+
assert payload["summary"]["model_asset_bytes"] == sum(
|
| 49 |
+
item["bytes"] for item in payload["files"] if item["model_asset"]
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
external = tmp_path / "weights.onnx.data"
|
| 53 |
+
external.write_bytes(external.read_bytes() + b"changed")
|
| 54 |
+
with pytest.raises(BundleManifestError, match="changed"):
|
| 55 |
+
verify_bundle_manifest(output)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def test_manifest_rejects_missing_external_data_and_empty_tokenizer(tmp_path: Path) -> None:
|
| 59 |
+
graph, tokenizer = _write_external_model(tmp_path)
|
| 60 |
+
(tmp_path / "weights.onnx.data").unlink()
|
| 61 |
+
with pytest.raises(BundleManifestError, match="external-data file does not exist"):
|
| 62 |
+
build_bundle_manifest(graph, tokenizer, tmp_path / "manifest.json")
|
| 63 |
+
|
| 64 |
+
graph, tokenizer = _write_external_model(tmp_path / "second")
|
| 65 |
+
for child in tokenizer.iterdir():
|
| 66 |
+
child.unlink()
|
| 67 |
+
with pytest.raises(BundleManifestError, match="contains no regular files"):
|
| 68 |
+
build_bundle_manifest(graph, tokenizer, tokenizer.parent / "manifest.json")
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def test_manifest_rejects_external_data_traversal(tmp_path: Path) -> None:
|
| 72 |
+
onnx = pytest.importorskip("onnx")
|
| 73 |
+
from onnx import TensorProto, helper
|
| 74 |
+
|
| 75 |
+
initializer = helper.make_tensor("weight", TensorProto.FLOAT, [1], [1.0])
|
| 76 |
+
initializer.data_location = TensorProto.EXTERNAL
|
| 77 |
+
initializer.external_data.add(key="location", value="../outside.onnx.data")
|
| 78 |
+
graph = helper.make_graph([], "test", [], [], [initializer])
|
| 79 |
+
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 18)])
|
| 80 |
+
graph_path = tmp_path / "model.onnx"
|
| 81 |
+
onnx.save_model(model, graph_path)
|
| 82 |
+
tokenizer = tmp_path / "tokenizer"
|
| 83 |
+
tokenizer.mkdir()
|
| 84 |
+
(tokenizer / "tokenizer.json").write_text("{}\n", encoding="utf-8")
|
| 85 |
+
|
| 86 |
+
with pytest.raises(BundleManifestError, match="traversal"):
|
| 87 |
+
build_bundle_manifest(graph_path, tokenizer, tmp_path / "manifest.json")
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def test_manifest_rejects_symlinked_tokenizer(tmp_path: Path) -> None:
|
| 91 |
+
graph, tokenizer = _write_external_model(tmp_path)
|
| 92 |
+
target = tmp_path / "outside.txt"
|
| 93 |
+
target.write_text("outside", encoding="utf-8")
|
| 94 |
+
(tokenizer / "escape.txt").symlink_to(target)
|
| 95 |
+
|
| 96 |
+
with pytest.raises(BundleManifestError, match="symlink"):
|
| 97 |
+
build_bundle_manifest(graph, tokenizer, tmp_path / "manifest.json")
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def test_source_snapshot_ignores_generated_python_caches(tmp_path: Path) -> None:
|
| 101 |
+
source = tmp_path / "source"
|
| 102 |
+
source.mkdir()
|
| 103 |
+
(source / "module.py").write_text("value = 1\n", encoding="utf-8")
|
| 104 |
+
ignored = source / "__pycache__"
|
| 105 |
+
ignored.mkdir()
|
| 106 |
+
(ignored / "module.cpython-313.pyc").write_bytes(b"first")
|
| 107 |
+
first = _snapshot_hash(source)
|
| 108 |
+
(ignored / "module.cpython-313.pyc").write_bytes(b"second")
|
| 109 |
+
(source / ".pytest_cache").mkdir()
|
| 110 |
+
(source / ".pytest_cache" / "state").write_bytes(b"cache")
|
| 111 |
+
assert _snapshot_hash(source) == first
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def test_source_snapshot_can_pin_an_explicit_file_list(tmp_path: Path) -> None:
|
| 115 |
+
source = tmp_path / "source"
|
| 116 |
+
source.mkdir()
|
| 117 |
+
(source / "module.py").write_text("value = 1\n", encoding="utf-8")
|
| 118 |
+
(source / "config.json").write_text("{}\n", encoding="utf-8")
|
| 119 |
+
|
| 120 |
+
first = _snapshot_hash(source, ["module.py"])
|
| 121 |
+
assert first["scope"] == "explicit"
|
| 122 |
+
assert first["paths"] == ["module.py"]
|
| 123 |
+
(source / "config.json").write_text('{"changed": true}\n', encoding="utf-8")
|
| 124 |
+
assert _snapshot_hash(source, ["module.py"]) == first
|
| 125 |
+
(source / "module.py").write_text("value = 2\n", encoding="utf-8")
|
| 126 |
+
assert _snapshot_hash(source, ["module.py"]) != first
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def test_source_snapshot_rejects_explicit_paths_outside_root(tmp_path: Path) -> None:
|
| 130 |
+
source = tmp_path / "source"
|
| 131 |
+
source.mkdir()
|
| 132 |
+
outside = tmp_path / "outside.py"
|
| 133 |
+
outside.write_text("value = 1\n", encoding="utf-8")
|
| 134 |
+
|
| 135 |
+
with pytest.raises(BundleManifestError, match="escapes"):
|
| 136 |
+
_snapshot_hash(source, [outside])
|
tests/test_calibration.py
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import json
|
| 3 |
+
import math
|
| 4 |
+
import tempfile
|
| 5 |
+
import unittest
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
from vons.calibration import (
|
| 9 |
+
ABSTAIN_ALL_THRESHOLD,
|
| 10 |
+
CALIBRATION_VERSION,
|
| 11 |
+
apply_calibration,
|
| 12 |
+
calibrate_report,
|
| 13 |
+
fit_abstention_threshold,
|
| 14 |
+
fit_abstention_threshold_details,
|
| 15 |
+
fit_temperature,
|
| 16 |
+
fit_temperature_details,
|
| 17 |
+
)
|
| 18 |
+
from vons.data import Example, write_jsonl
|
| 19 |
+
from vons.evaluation import METRIC_VERSION, Prediction, nll, summarize, write_report
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _example(identifier: str, label: str | None, answerable: bool, split: str = "calibration") -> Example:
|
| 23 |
+
return Example(
|
| 24 |
+
id=identifier,
|
| 25 |
+
task_group="calibration",
|
| 26 |
+
state="state",
|
| 27 |
+
question="choose",
|
| 28 |
+
options=("a", "b"),
|
| 29 |
+
label=label,
|
| 30 |
+
answerable=answerable,
|
| 31 |
+
split=split,
|
| 32 |
+
provenance={},
|
| 33 |
+
metadata={"scenario_id": identifier},
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _reject_constant(value: str) -> float:
|
| 38 |
+
raise ValueError(f"non-strict JSON constant {value}")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class CalibrationTests(unittest.TestCase):
|
| 42 |
+
def test_temperature_fit_finds_interior_optimum_and_does_not_increase_nll(self) -> None:
|
| 43 |
+
examples = [_example(f"r{index}", "a" if index < 3 else "b", True) for index in range(4)]
|
| 44 |
+
predictions = [Prediction(f"r{index}", "a", {"a": 0.6, "b": 0.4}, 0.6, False) for index in range(4)]
|
| 45 |
+
|
| 46 |
+
details = fit_temperature_details(examples, predictions)
|
| 47 |
+
calibrated = apply_calibration(examples, predictions, temperature=details.temperature, abstention_threshold=0.0)
|
| 48 |
+
|
| 49 |
+
self.assertAlmostEqual(details.temperature, math.log(1.5) / math.log(3.0), places=6)
|
| 50 |
+
self.assertEqual(details.rows, 4)
|
| 51 |
+
self.assertFalse(details.at_lower_bound)
|
| 52 |
+
self.assertFalse(details.at_upper_bound)
|
| 53 |
+
self.assertEqual(fit_temperature(examples, predictions), details.temperature)
|
| 54 |
+
self.assertLess(nll(examples, calibrated), nll(examples, predictions))
|
| 55 |
+
|
| 56 |
+
def test_temperature_bound_hit_is_flagged(self) -> None:
|
| 57 |
+
examples = [_example(f"s{index}", "a", True) for index in range(4)]
|
| 58 |
+
predictions = [Prediction(f"s{index}", "a", {"a": 0.9, "b": 0.1}, 0.9, False) for index in range(4)]
|
| 59 |
+
|
| 60 |
+
details = fit_temperature_details(examples, predictions)
|
| 61 |
+
|
| 62 |
+
self.assertTrue(details.at_lower_bound)
|
| 63 |
+
self.assertAlmostEqual(details.temperature, 0.05)
|
| 64 |
+
|
| 65 |
+
def test_temperature_scaling_preserves_true_zero_support(self) -> None:
|
| 66 |
+
examples = [_example("zero", "a", True)]
|
| 67 |
+
predictions = [Prediction("zero", "a", {"a": 1.0, "b": 0.0}, 1.0, False)]
|
| 68 |
+
|
| 69 |
+
calibrated = apply_calibration(examples, predictions, temperature=20.0, abstention_threshold=0.0)
|
| 70 |
+
|
| 71 |
+
self.assertEqual(calibrated[0].probabilities, {"a": 1.0, "b": 0.0})
|
| 72 |
+
|
| 73 |
+
def test_invalid_probability_vectors_fail_closed(self) -> None:
|
| 74 |
+
examples = [_example("x", "a", True)]
|
| 75 |
+
for probabilities in ({"a": 0.5, "b": 0.2}, {"a": math.nan, "b": 1.0}, {"a": -0.5, "b": 1.5}):
|
| 76 |
+
with self.subTest(probabilities=probabilities), self.assertRaises(ValueError):
|
| 77 |
+
apply_calibration(
|
| 78 |
+
examples,
|
| 79 |
+
[Prediction("x", "a", probabilities, 0.5, False)],
|
| 80 |
+
temperature=1.0,
|
| 81 |
+
abstention_threshold=0.0,
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
def test_calibration_recomputes_argmax_and_never_covers_without_choice(self) -> None:
|
| 85 |
+
examples = [_example("was_abstained", "a", True), _example("stale_choice", "a", True)]
|
| 86 |
+
predictions = [
|
| 87 |
+
Prediction("was_abstained", None, {"a": 0.6, "b": 0.4}, 0.42, True),
|
| 88 |
+
Prediction("stale_choice", "b", {"a": 0.6, "b": 0.4}, 0.6, False),
|
| 89 |
+
]
|
| 90 |
+
|
| 91 |
+
calibrated = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=0.2)
|
| 92 |
+
|
| 93 |
+
self.assertEqual([(item.choice, item.abstained) for item in calibrated], [("a", False), ("a", False)])
|
| 94 |
+
|
| 95 |
+
def test_answerability_gate_is_preserved_and_labels_do_not_gate(self) -> None:
|
| 96 |
+
examples = [_example("low_answerability", "a", True), _example("unanswerable", None, False)]
|
| 97 |
+
predictions = [
|
| 98 |
+
Prediction("low_answerability", "a", {"a": 0.6, "b": 0.4}, 0.2, False),
|
| 99 |
+
Prediction("unanswerable", "a", {"a": 0.6, "b": 0.4}, 0.6, False),
|
| 100 |
+
]
|
| 101 |
+
|
| 102 |
+
default = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=0.0)
|
| 103 |
+
relaxed = apply_calibration(
|
| 104 |
+
examples,
|
| 105 |
+
predictions,
|
| 106 |
+
temperature=1.0,
|
| 107 |
+
abstention_threshold=0.0,
|
| 108 |
+
answerability_threshold=0.3,
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
self.assertEqual((default[0].choice, default[0].abstained), (None, True))
|
| 112 |
+
self.assertEqual((default[1].choice, default[1].abstained), ("a", False))
|
| 113 |
+
self.assertEqual((relaxed[0].choice, relaxed[0].abstained), ("a", False))
|
| 114 |
+
|
| 115 |
+
def test_failed_and_absent_distributions_remain_abstained(self) -> None:
|
| 116 |
+
examples = [_example("failed", "a", True), _example("absent", "a", True)]
|
| 117 |
+
predictions = [
|
| 118 |
+
Prediction("failed", None, {}, 0.0, False, error="timeout"),
|
| 119 |
+
Prediction("absent", "a", {}, 0.9, False),
|
| 120 |
+
]
|
| 121 |
+
|
| 122 |
+
calibrated = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=0.0)
|
| 123 |
+
|
| 124 |
+
self.assertEqual((calibrated[0].choice, calibrated[0].abstained, calibrated[0].error), (None, True, "timeout"))
|
| 125 |
+
self.assertEqual((calibrated[1].choice, calibrated[1].abstained, calibrated[1].probabilities), (None, True, {}))
|
| 126 |
+
|
| 127 |
+
def test_confidence_above_max_probability_fails_closed(self) -> None:
|
| 128 |
+
examples = [_example("x", "a", True)]
|
| 129 |
+
with self.assertRaisesRegex(ValueError, "max_probability"):
|
| 130 |
+
apply_calibration(
|
| 131 |
+
examples,
|
| 132 |
+
[Prediction("x", "a", {"a": 0.5, "b": 0.5}, 0.9, False)],
|
| 133 |
+
temperature=1.0,
|
| 134 |
+
abstention_threshold=0.0,
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
def test_threshold_fit_uses_argmax_and_counts_unanswerable_as_errors(self) -> None:
|
| 138 |
+
examples = [_example("correct", "a", True), _example("stale_choice", "a", True), _example("unknown", None, False)]
|
| 139 |
+
predictions = [
|
| 140 |
+
Prediction("correct", "a", {"a": 0.9, "b": 0.1}, 0.9, False),
|
| 141 |
+
Prediction("stale_choice", "b", {"a": 0.6, "b": 0.4}, 0.58, False),
|
| 142 |
+
Prediction("unknown", "a", {"a": 0.55, "b": 0.45}, 0.5, False),
|
| 143 |
+
]
|
| 144 |
+
|
| 145 |
+
details = fit_abstention_threshold_details(examples, predictions, target_risk=0.0)
|
| 146 |
+
calibrated = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=details.threshold)
|
| 147 |
+
|
| 148 |
+
self.assertAlmostEqual(details.threshold, 0.58)
|
| 149 |
+
self.assertEqual((details.rows, details.covered_rows, details.risk, details.abstain_all), (3, 2, 0.0, False))
|
| 150 |
+
self.assertEqual(fit_abstention_threshold(examples, predictions, target_risk=0.0), details.threshold)
|
| 151 |
+
self.assertEqual([item.abstained for item in calibrated], [False, False, True])
|
| 152 |
+
|
| 153 |
+
def test_all_abstain_sentinel_covers_nothing_even_at_confidence_one(self) -> None:
|
| 154 |
+
examples = [_example("wrong", "a", True)]
|
| 155 |
+
predictions = [Prediction("wrong", "b", {"a": 0.0, "b": 1.0}, 1.0, False)]
|
| 156 |
+
|
| 157 |
+
threshold = fit_abstention_threshold(examples, predictions, target_risk=0.0)
|
| 158 |
+
at_one = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=1.0)[0]
|
| 159 |
+
sentinel = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=threshold)[0]
|
| 160 |
+
|
| 161 |
+
self.assertEqual(threshold, ABSTAIN_ALL_THRESHOLD)
|
| 162 |
+
self.assertFalse(at_one.abstained)
|
| 163 |
+
self.assertEqual((sentinel.choice, sentinel.abstained), (None, True))
|
| 164 |
+
for invalid in (1.5, -0.1, math.nan):
|
| 165 |
+
with self.subTest(threshold=invalid), self.assertRaisesRegex(ValueError, "abstention_threshold"):
|
| 166 |
+
apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=invalid)
|
| 167 |
+
|
| 168 |
+
def test_unknown_or_duplicate_prediction_ids_fail_closed(self) -> None:
|
| 169 |
+
examples = [_example("a", "a", True)]
|
| 170 |
+
with self.assertRaisesRegex(ValueError, "unknown example id"):
|
| 171 |
+
apply_calibration(
|
| 172 |
+
examples,
|
| 173 |
+
[Prediction("ghost", "a", {"a": 1.0}, 1.0, False)],
|
| 174 |
+
temperature=1.0,
|
| 175 |
+
abstention_threshold=0.0,
|
| 176 |
+
)
|
| 177 |
+
with self.assertRaisesRegex(ValueError, "duplicate prediction"):
|
| 178 |
+
fit_abstention_threshold(examples, [Prediction("a", "a", {"a": 1.0}, 1.0, False)] * 2)
|
| 179 |
+
|
| 180 |
+
def test_calibrate_report_records_provenance_and_rejects_identity_mismatch(self) -> None:
|
| 181 |
+
config = {
|
| 182 |
+
"model_id": "vons-test",
|
| 183 |
+
"backend": "direct",
|
| 184 |
+
"seed": 7,
|
| 185 |
+
"encoder": {"name": "encoder", "revision": "r1"},
|
| 186 |
+
"answerability_threshold": 0.8,
|
| 187 |
+
}
|
| 188 |
+
calibration_rows = [_example("c1", "a", True), _example("c2", "b", True), _example("c3", None, False)]
|
| 189 |
+
calibration_predictions = [
|
| 190 |
+
Prediction("c1", "a", {"a": 0.9, "b": 0.1}, 0.9, False),
|
| 191 |
+
Prediction("c2", "b", {"a": 0.3, "b": 0.7}, 0.63, False),
|
| 192 |
+
Prediction("c3", None, {"a": 0.6, "b": 0.4}, 0.3, True),
|
| 193 |
+
]
|
| 194 |
+
test_rows = [_example("t1", "a", True, "test"), _example("t2", "b", True, "test"), _example("t3", None, False, "test")]
|
| 195 |
+
test_predictions = [
|
| 196 |
+
Prediction("t1", "a", {"a": 0.8, "b": 0.2}, 0.8, False),
|
| 197 |
+
Prediction("t2", "b", {"a": 0.4, "b": 0.6}, 0.42, False),
|
| 198 |
+
Prediction("t3", None, {}, 0.0, False, error="timeout"),
|
| 199 |
+
]
|
| 200 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 201 |
+
root = Path(directory)
|
| 202 |
+
calibration_input, calibration_report = root / "calibration.jsonl", root / "calibration.json"
|
| 203 |
+
test_input, test_report, output = root / "test.jsonl", root / "test.json", root / "calibrated.json"
|
| 204 |
+
write_jsonl(calibration_input, calibration_rows)
|
| 205 |
+
write_jsonl(test_input, test_rows)
|
| 206 |
+
write_report(calibration_report, config=config, summary=summarize(calibration_rows, calibration_predictions), predictions=calibration_predictions)
|
| 207 |
+
write_report(test_report, config=config, summary=summarize(test_rows, test_predictions), predictions=test_predictions)
|
| 208 |
+
arguments = {
|
| 209 |
+
"calibration_input": calibration_input,
|
| 210 |
+
"calibration_report": calibration_report,
|
| 211 |
+
"test_input": test_input,
|
| 212 |
+
"test_report": test_report,
|
| 213 |
+
}
|
| 214 |
+
|
| 215 |
+
calibrate_report(**arguments, output=output, target_risk=0.0)
|
| 216 |
+
payload = json.loads(output.read_text(encoding="utf-8"), parse_constant=_reject_constant)
|
| 217 |
+
|
| 218 |
+
fit = payload["calibration"]
|
| 219 |
+
self.assertEqual(fit["version"], CALIBRATION_VERSION)
|
| 220 |
+
self.assertEqual(fit["answerability_threshold"], 0.8)
|
| 221 |
+
self.assertEqual((fit["temperature_fit_rows"], fit["threshold_fit_rows"], fit["rows"]), (2, 3, 3))
|
| 222 |
+
self.assertTrue(fit["temperature_at_lower_bound"])
|
| 223 |
+
self.assertLessEqual(fit["nll_after"], fit["nll_before"])
|
| 224 |
+
self.assertEqual(payload["inputs"]["test_report"]["sha256"], hashlib.sha256(test_report.read_bytes()).hexdigest())
|
| 225 |
+
self.assertEqual(payload["model_identity"]["model_id"], "vons-test")
|
| 226 |
+
self.assertTrue(any("checkpoint_sha256 unavailable" in item for item in payload["limitations"]))
|
| 227 |
+
self.assertEqual(payload["summary"]["metric_version"], METRIC_VERSION)
|
| 228 |
+
decisions = {item["example_id"]: (item["choice"], item["abstained"]) for item in payload["predictions"]}
|
| 229 |
+
self.assertEqual(decisions, {"t1": ("a", False), "t2": (None, True), "t3": (None, True)})
|
| 230 |
+
|
| 231 |
+
mismatched = root / "mismatched.json"
|
| 232 |
+
write_report(mismatched, config={**config, "model_id": "other"}, summary={}, predictions=test_predictions)
|
| 233 |
+
with self.assertRaisesRegex(ValueError, "model_id"):
|
| 234 |
+
calibrate_report(**{**arguments, "test_report": mismatched}, output=root / "rejected.json")
|
| 235 |
+
with self.assertRaisesRegex(ValueError, "overwrite"):
|
| 236 |
+
calibrate_report(**arguments, output=test_report)
|
| 237 |
+
self.assertFalse((root / "rejected.json").exists())
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
if __name__ == "__main__":
|
| 241 |
+
unittest.main()
|
tests/test_contract.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import unittest
|
| 2 |
+
|
| 3 |
+
from vons.contract import (
|
| 4 |
+
Backend,
|
| 5 |
+
DecisionRequest,
|
| 6 |
+
Question,
|
| 7 |
+
QuestionType,
|
| 8 |
+
derive_kasi_action,
|
| 9 |
+
validate_request,
|
| 10 |
+
)
|
| 11 |
+
from vons.kasi import KASIAdapter, KASIProposal
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class ContractTests(unittest.TestCase):
|
| 15 |
+
def test_valid_request(self) -> None:
|
| 16 |
+
validate_request(DecisionRequest("A weather tool is available.", (Question("next", QuestionType.CHOICE, "Choose.", ("call", "clarify")),), Backend.DIRECT, 7))
|
| 17 |
+
|
| 18 |
+
def test_duplicate_questions_rejected(self) -> None:
|
| 19 |
+
request = DecisionRequest("state", (Question("q", QuestionType.CHOICE, "pick", ("a", "b")), Question("q", QuestionType.CHOICE, "pick", ("a", "b"))))
|
| 20 |
+
with self.assertRaises(ValueError):
|
| 21 |
+
validate_request(request)
|
| 22 |
+
|
| 23 |
+
def test_kasi_confirm_is_wrapper_derived(self) -> None:
|
| 24 |
+
self.assertEqual(derive_kasi_action(proposed_calls=({"name": "unlock"},), confidence=0.99, risk="high", confirmed_tools=set(), tool_policy={"unlock": "high"}).value, "confirm")
|
| 25 |
+
self.assertEqual(derive_kasi_action(proposed_calls=({"name": "read"},), confidence=0.99, risk="low", confirmed_tools=set(), tool_policy={"read": "low"}).value, "call")
|
| 26 |
+
|
| 27 |
+
def test_unregistered_tool_cannot_call(self) -> None:
|
| 28 |
+
action = derive_kasi_action(proposed_calls=({"name": "read"},), confidence=0.99, risk="low", tool_policy={})
|
| 29 |
+
self.assertEqual(action.value, "refuse")
|
| 30 |
+
|
| 31 |
+
def test_kasi_response_and_low_confidence(self) -> None:
|
| 32 |
+
adapter = KASIAdapter(tool_policy={"read_weather": "low"})
|
| 33 |
+
response = adapter.decide(KASIProposal(response="The answer is ready.", confidence=0.9))
|
| 34 |
+
self.assertEqual(response.action.value, "respond")
|
| 35 |
+
clarify = adapter.decide(KASIProposal(calls=(), confidence=0.2))
|
| 36 |
+
self.assertEqual(clarify.action.value, "clarify")
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
if __name__ == "__main__":
|
| 40 |
+
unittest.main()
|