diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..dc78cfe373385821b433d8a6d41766f0d54a2e99 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,35 +1,12 @@ -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text -*.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text +* text=auto +*.py text eol=lf +*.ts text eol=lf +*.mjs text eol=lf +*.md text eol=lf +*.json text eol=lf +*.yml text eol=lf +*.cff text eol=lf +*.pdf binary +*.png binary +docs/publications/vons-paper-v1.0.pdf filter=lfs diff=lfs merge=lfs -text +docs/publications/vons-tech-report-v1.0.1.pdf filter=lfs diff=lfs merge=lfs -text diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 0000000000000000000000000000000000000000..1f05e80d7306c7bd1da3d361e1a7ff8e3a5ae95e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,31 @@ +name: Bug report +description: Report unexpected behavior with a minimal safe reproduction. +title: "[Bug] " +body: + - type: markdown + attributes: + value: | + For security issues, contact oswarld@inlevel9.com privately with a brief, + nonsensitive description first. Do not post secrets or restricted inputs. + English and Korean reports are welcome. + - type: textarea + id: environment + attributes: + label: Version and environment + description: Source revision or release manifest, OS, runtime version and optional model/provider identity. + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Minimal reproduction + description: Exact command and preferably synthetic input; remove personal information and credentials. + validations: + required: true + - type: textarea + id: behavior + attributes: + label: Expected and actual behavior + description: Include safe error output, frequency and any workaround. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/experience.yml b/.github/ISSUE_TEMPLATE/experience.yml new file mode 100644 index 0000000000000000000000000000000000000000..a46b147b70492778ced7f5aa16a770b5b3b3b99f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/experience.yml @@ -0,0 +1,55 @@ +name: Experience report +description: Share what worked, failed or remained unclear when trying Vons. +title: "[Experience] " +body: + - type: markdown + attributes: + value: | + Positive and negative experiences are welcome, in English or Korean. + This is a public report once the repository is public. Use synthetic or + redacted examples. Do not upload secrets, private prompts, customer traces, + restricted data or model weights. A report is not benchmark validation. + - type: dropdown + id: path + attributes: + label: What did you try? + options: + - Documentation or installation + - Deterministic contract / KASI adapter + - Synthetic data walkthrough + - Model inference with separately supplied assets + validations: + required: true + - type: input + id: version + attributes: + label: Version or source identity + description: Commit, release manifest digest or explicitly unknown; include asset digest for inference. + validations: + required: true + - type: textarea + id: context + attributes: + label: Scenario and environment + description: Your goal, OS, Python or Node version, device, runtime and provider. Omit personal details. + validations: + required: true + - type: textarea + id: outcome + attributes: + label: Expected and observed behavior + description: Include the command or steps, what helped, what failed and limitations. + validations: + required: true + - type: textarea + id: counts + attributes: + label: Counts and optional safe evidence + description: Attempted / successful / abstained / failed with definitions; explain overlaps. Use unavailable where unmeasured. Link only reviewed evidence. + - type: checkboxes + id: privacy + attributes: + label: Public content check + options: + - label: I reviewed this report for secrets, personal information and restricted material, and may share its contents. + required: true diff --git a/.github/ISSUE_TEMPLATE/reproduction.yml b/.github/ISSUE_TEMPLATE/reproduction.yml new file mode 100644 index 0000000000000000000000000000000000000000..eb1cff80b6e1fe52e734b48828e189a6682dda00 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/reproduction.yml @@ -0,0 +1,50 @@ +name: Reproduction report +description: Report a scoped attempt to reproduce a documented behavior or result. +title: "[Reproduction] " +body: + - type: markdown + attributes: + value: | + An unsuccessful reproduction is useful. Keep raw records locally and + publish only permitted, reviewed aggregates or synthetic examples. + English and Korean reports are welcome. Missing evidence is unavailable. + - type: textarea + id: claim + attributes: + label: Claim and source + description: Link the exact document/version and state the claim's scope and expected result. + validations: + required: true + - type: textarea + id: setup + attributes: + label: Versions, inputs and environment + description: Source revision or manifest, model/tokenizer hashes if used, permitted input provenance, split, seed, OS/device/runtime/provider. Do not attach restricted inputs. + validations: + required: true + - type: textarea + id: procedure + attributes: + label: Commands and measurement procedure + description: Include warm-up, repetitions, timing or memory scope and analysis choices where applicable. + validations: + required: true + - type: textarea + id: results + attributes: + label: Observed results and denominators + description: Attempts, successes, abstentions, failures and unavailable values; include definitions, mismatches and limitations. Separate targets from measurements. + validations: + required: true + - type: textarea + id: evidence + attributes: + label: Optional permitted evidence + description: Safe aggregate report or synthetic fixture with hashes; state what cannot be shared or independently checked. + - type: checkboxes + id: privacy + attributes: + label: Public content check + options: + - label: I reviewed the report and attachments for secrets, personal information and restricted material, and may share them. + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000000000000000000000000000000000..bf051191d620e5906a1c27590d0efb2cdd84afc9 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## Problem and change + +Describe the behavior before and after this change, and link the related issue. + +## Verification + +List commands run and their results. State skips, unavailable evidence and known +limitations. Separate measured values from proposed targets. + +## Review notes + +- Explain any public contract, dependency, data or license implications. +- Identify third-party material and its provenance when applicable. +- Keep raw restricted data, model weights, secrets and private prompts out. +- Update the release allowlist for reviewed new public files. +- Preserve the separation between model decisions and host execution/consent. diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000000000000000000000000000000000000..55812db0ef9cd75139ab177a3acd4ececf36bdd8 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,61 @@ +name: Source checks + +on: + push: + paths: + - '.github/workflows/checks.yml' + - 'vons/**' + - 'tools/**' + - 'tests/**' + - 'configs/**' + - 'sdk/**' + - 'pyproject.toml' + - '.gitignore' + pull_request: + paths: + - '.github/workflows/checks.yml' + - 'vons/**' + - 'tools/**' + - 'tests/**' + - 'configs/**' + - 'sdk/**' + - 'pyproject.toml' + - '.gitignore' + workflow_dispatch: + +concurrency: + group: source-checks-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: python -m pip install -e '.[dev]' numpy onnx onnxruntime + - run: python tools/prepare_public_release.py + - run: ruff check . + - run: python -m pytest -q + typescript: + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdk/typescript + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: sdk/typescript/package-lock.json + - run: npm ci + - run: npm test + - run: npm run build:browser + - run: npm run build:benchmark + - run: npm run build:parity diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..c3573abb793c7ae59fd52a1e82153fd800207a5a --- /dev/null +++ b/.gitignore @@ -0,0 +1,148 @@ +# Public source allowlist. New root paths stay local until explicitly reviewed. +/* +!/.gitignore +!/.gitattributes +!/.github/ +!/README.md +!/CONTRIBUTING.md +!/CITATION.cff +!/LICENSE +!/NOTICE +!/pyproject.toml +!/configs/ +!/docs/ +!/sdk/ +!/tests/ +!/tools/ +!/vons/ + +# Internal planning, historical claims and agent/session records stay local. +/docs/* +!/docs/KASI_INTEGRATION.md +!/docs/MODEL_CARD.md +!/docs/PUBLIC_RELEASE.md +!/docs/HUGGING_FACE.md +!/docs/LICENSING.md +!/docs/COMMUNITY.md +!/docs/HUB_PUBLISHING.md +!/docs/publications/ +!/docs/assets/ +/docs/publications/* +!/docs/publications/README.md +!/docs/publications/LICENSE.md +!/docs/publications/ARXIV.md +/docs/assets/* +!/docs/assets/inlevel9-signature.png +!/docs/assets/inlevel9-signature-dark.png +/configs/* +!/configs/pilot.json +!/configs/pilot-diffusion.json +!/configs/public-release.json +/sdk/* +!/sdk/typescript/ +/sdk/typescript/* +!/sdk/typescript/README.md +!/sdk/typescript/LICENSE +!/sdk/typescript/package.json +!/sdk/typescript/package-lock.json +!/sdk/typescript/src/ +!/sdk/typescript/test/ +!/sdk/typescript/demo/ + +# Raw/generated data, weights, evidence packets and local publications. +data/ +artifacts/ +reports/ +research/ +models/ +weights/ +checkpoints/ +outputs/ +releases/ +private/ +restricted/ +*.onnx +*.onnx.* +*.safetensors +*.gguf +*.ggml +*.pt +*.pth +*.ckpt +*.bin +*.h5 +*.hdf5 +*.npz +*.npy +*.pkl +*.pickle +*.parquet +*.arrow +*.jsonl +*.zip +*.tar +*.tar.* +*.tgz +*.7z +*.dmg +*.pdf +# Only the exact reviewed publication bytes are eligible for staging. +!/docs/publications/vons-paper-v1.0.pdf +!/docs/publications/vons-tech-report-v1.0.1.pdf + +# Credentials, machine-specific config, private prompts and service caches. +.env +.env.* +*.env +*.pem +*.key +*.p12 +*.pfx +*.jks +credentials*.json +secrets*.json +service-account*.json +token.json +.npmrc +.pypirc +.netrc +.aws/ +.ssh/ +.config/ +.cache/ +.huggingface/ +.claude/ +.codex/ +.cursor/ +.trae/ +.idea/ +.vscode/ +*.local.* +*.log +*.sqlite* +*.db +*.bak +*.swp +*~ +.git_backup/ + +# Environments, dependencies and generated browser bundles. +.venv/ +venv/ +node_modules/ +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.tox/ +.coverage* +htmlcov/ +coverage/ +build/ +dist/ +*.egg-info/ +/sdk/typescript/demo/*.js +/sdk/typescript/demo/*.js.map +.DS_Store +Thumbs.db diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000000000000000000000000000000000..61a0247bffea10ebe871b5432362011ff53ba158 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,11 @@ +cff-version: 1.2.0 +message: "If you use this research source, cite this version and the accompanying publications when available." +type: software +title: "Vons: Compact Local Decision Models for Frontier-Agent Workflows" +version: 0.1.0 +authors: + - family-names: Ahn + given-names: Kwangseob + affiliation: "INLEVEL9 / SEJONG UNIV." + email: oswarld@inlevel9.com +abstract: "Research source for bounded local candidate selection and abstention, with host-owned execution and consent." diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..49760ddc35fdc500adbd98cb70c8e1af0d53b88c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,81 @@ +# Contributing to Vons + +Help make bounded local decisions easier to understand, reproduce and improve. +Questions, unsuccessful trials, documentation fixes and small code changes are +welcome. Start with the [community guide](docs/COMMUNITY.md). + +## Choose a contribution + +- Share an experience using the **Experience report** issue form. Describe what + happened, including failures; a positive result is not required. +- Submit a **Reproduction report** with exact commands, versions, denominators + and the scope of the claim you checked. +- Use **Bug report** for a minimal, preferably synthetic, reproduction. +- Propose a focused change through a pull request. Discuss changes to the public + contract, data handling, model architecture or dependencies before implementing + them. Small documentation corrections can go directly to a pull request. + +The intended repository is [inlevel9-com/Jons](https://github.com/inlevel9-com/Jons). +Its URL spelling differs from the Vons project name. It was private and empty +when these instructions were prepared; public participation starts after the +maintainer publishes the reviewed source and verifies the issue forms. + +## Work locally + +Use Python 3.10 or later. From a source checkout: + +```sh +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -e '.[dev]' +python -m pytest -q +ruff check . +python tools/prepare_public_release.py +``` + +On Windows, activate with `.venv\Scripts\Activate.ps1` in PowerShell. Tests for +optional training/export dependencies may skip explicitly. Do not download +restricted data or model weights simply to remove a skip. For TypeScript work, +use the [SDK checks](sdk/typescript/README.md). + +Keep one purpose per change. Add behavior-focused regression coverage for fixes. +Report checks you ran, checks you could not run, and the reason. English is used +for code, comments and repository documentation; community questions and +experience reports may be in English or Korean. + +## Preserve the boundaries + +The general decision contract and the KASI compatibility adapter are separate. +Model output never executes a tool, grants consent or authorizes a high-risk +action. Self-reported confidence is not a calibrated probability. Missing +measurements stay unavailable; design targets are not achieved results. + +Use synthetic or redacted examples in public reports. Do not attach credentials, +private prompts, customer traces, personal data, restricted benchmark rows, +model weights or private evidence archives. Keep your own raw records locally; +share only material you are entitled to disclose. Contact +[oswarld@inlevel9.com](mailto:oswarld@inlevel9.com) before disclosing a security +issue; the first message should describe the issue without credentials or +sensitive attachments. + +New public files must be reviewed and added to `configs/public-release.json` +and, where necessary, `.gitignore`. Never force-add ignored research material. + +## Review and attribution + +Maintainers review scope, correctness, evidence and redistribution rights before +merging. An issue, testimonial or model-assisted review is not acceptance, +independent replication or peer review. Review may request changes or decline a +proposal; no response-time or merge guarantee is implied. + +Be specific and respectful. Critique claims and code rather than people. Spam, +harassment, fabricated evidence and exposed private information may be removed; +technical criticism and negative results are welcome. Corrections should keep +the original claim and explain what changed whenever it is safe to do so. + +The software uses the [Vons Community and Commercial License](LICENSE), with +separate [article terms](docs/publications/LICENSE.md). Submitting an issue does +not grant software-use rights or permission to use it as a promotional quote. +Code contributions are reviewed for compatibility with the repository license; +identify third-party material and obtain any required employer permission. +No copyright assignment or separate contributor agreement is created here. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..02fdf205459235265f92d5fe07a7f143ca6c1ded --- /dev/null +++ b/LICENSE @@ -0,0 +1,104 @@ +Vons Community and Commercial License 1.0 +SPDX-License-Identifier: LicenseRef-Vons-Community-Commercial-1.0 + +1. Scope and definitions + +"Licensor" means each copyright holder who makes material available under this +license. "Software" means the Vons-owned source code, configuration, tests and +supporting software documentation accompanied by this license, including copies +and modifications. "You" means the individual or legal entity exercising the +permissions below, including persons acting on that entity's behalf. + +This license does not cover material with a separate license notice. In +particular, the paper and Tech Report and their original research figures are +licensed separately under CC BY 4.0. Third-party dependencies, upstream model +assets, datasets, trademarks and logos are not relicensed by this document. +No model weights or benchmark datasets are supplied with this source release. + +"Noncommercial Use" means personal learning, independent noncommercial +research, or noncommercial education, research or public-benefit activity by a +nonprofit or educational institution, where the activity is not undertaken for +commercial advantage, monetary compensation, or on behalf of a business. +Nonprofit or educational status alone does not make commercial activity +Noncommercial Use. Unrestricted donations that do not buy a service or benefit +do not by themselves make an otherwise qualifying activity commercial. + +"Commercial Use" includes paid consulting or client work; use in or to support +a monetized product, service, SaaS, API, hosted or managed service; internal +business operations; and evaluation, research or deployment by, for, or on +behalf of a for-profit enterprise. Contractors and affiliates acting for such +an enterprise require the same permission. Personal use by an employee outside +their employment, without business benefit, may qualify as Noncommercial Use. + +2. Free community permission + +Subject to this license, each Licensor grants You a worldwide, nonexclusive, +royalty-free copyright license to use, reproduce, study, modify, display and +distribute the Software solely for Noncommercial Use. You may provide access +to an otherwise qualifying noncommercial service under these conditions. +Only rights that a Licensor is entitled to grant are included. + +3. Redistribution and attribution + +When distributing copies or substantial portions of the Software, including +modified copies or a compiled form, You must: + +(a) retain this license, copyright notices and attribution notices; +(b) clearly identify Your modifications and preserve their provenance; +(c) identify the Vons version or source revision used where available; and +(d) convey these same use restrictions for the covered Software, without + representing that You can grant Commercial Use rights. + +Do not remove notices, misrepresent origin, or imply endorsement by a Licensor, +INLEVEL9, Sejong University or any other named institution. For research that +uses Vons, include an appropriate citation to the software and relevant paper +or Tech Report when reporting results. An unavailable DOI is not required. +Attribution or citation does not authorize an otherwise restricted use. + +4. Enterprise and commercial permission + +Commercial Use is not granted by this license. It requires a separate written +commercial agreement from the relevant Licensor before the activity begins. +There is no automatic enterprise evaluation or free-trial exception. Contact +oswarld@inlevel9.com to arrange the scope, fees and permitted entities. +This license sets no price, billing unit, service level or support commitment. +A commercial agreement applies only to the parties and scope it identifies. + +5. Outputs and independent rights + +This license claims no ownership or royalty over Your inference outputs merely +because they were produced using the Software. Outputs do not authorize an +unlicensed deployment and may contain material subject to independent rights. +Your inputs remain subject to their own rights and permissions. + +No trademark, logo, certification or endorsement rights are granted, except +the descriptive use needed to comply with the notices above. No patent license +is granted by this document. Statutory exceptions and other independently +obtained permissions are unaffected. This license does not revoke rights +previously granted under another license for an earlier copy, and does not +restrict separately licensed third-party components. + +6. Termination and cure + +Your permissions terminate automatically if You materially breach this +license. They are reinstated when the breach is cured within 30 days after +You first discover it, provided this is Your first breach. Otherwise written +reinstatement from the relevant Licensor is required. Termination does not +affect compliant recipients' direct permissions or obligations that by their +nature survive, including the notices and disclaimers below. + +7. Warranty disclaimer and liability + +TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE SOFTWARE IS PROVIDED +"AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND +NONINFRINGEMENT. THE LICENSORS ARE NOT LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY ARISING FROM THE SOFTWARE OR ITS USE, WHETHER IN CONTRACT, TORT OR +OTHERWISE. NOTHING HERE EXCLUDES LIABILITY THAT CANNOT LAWFULLY BE EXCLUDED. + +8. General + +Rights not expressly granted are reserved. No later license version applies +automatically. If a provision is unenforceable, the remaining provisions remain +in effect to the extent permitted by law. This is a source-available license +with use restrictions; it is not an OSI-approved open-source license. diff --git a/README.md b/README.md index 154df8298fab5ecf322016157858e08cd1bccbe1..83d1d2fa49547f6cad203d95390f9a10fa66343a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,93 @@ --- -license: apache-2.0 +license: other +license_name: vons-community-commercial-1.0 +license_link: LICENSE +language: + - en +tags: + - vons + - research + - candidate-selection + - onnx --- + +![INLEVEL9](docs/assets/inlevel9-signature.png) + +# Vons + +**Plan globally. Decide locally. Keep the host in control.** + +Compact local decision models for frontier-agent workflows. + +**Kwangseob Ahn** · **INLEVEL9 / SEJONG UNIV.** · **oswarld@inlevel9.com** + +## Source preview + +This repository contains research source and documentation. It does not contain +pretrained model weights, tokenizer assets or datasets. Hosted inference and +`from_pretrained` loading are not available from this source preview. + +Vons studies Direct and Diffusion candidate scoring with abstention. The host +owns execution, permission and consent. Model output never authorizes an action. +English-first operation and a model asset budget under 64 MiB are design targets; +they are not general quality or complete browser-package guarantees. + +See [the model card](docs/MODEL_CARD.md), +[the TypeScript SDK](sdk/typescript/README.md), +[the paper and Tech Report](docs/publications/README.md), and +[release preparation](docs/PUBLIC_RELEASE.md). + +## Read, try and contribute + +This source preview is distributed through [INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons). +Source and contribution reports belong in +[inlevel9-com/Vons](https://github.com/inlevel9-com/Vons). The reviewed paper and Tech Report are included as PDF downloads below. The software remains a 0.1.0 +research preview. No pretrained weights or working Space are announced. + +Follow the [community guide](docs/COMMUNITY.md) for the no-weight walkthrough, +experience and reproduction reports, and the +[contribution guide](CONTRIBUTING.md) for review expectations. The paper is +being prepared for [arXiv](docs/publications/ARXIV.md). User reports are not +independent research validation. + +Maintainers can use the [Hub upload guide](docs/HUB_PUBLISHING.md) to prepare +an organization card, source preview and later a separately validated Space. + +## Paper and Tech Report + +| Publication | Download | +| --- | --- | +| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows — paper v1.0, final editorial revision | [PDF · 15 pages](docs/publications/vons-paper-v1.0.pdf) | +| Vons: Compact Decision Models for Frontier-Agent Workflows — Technical Report v1.0.1 | [PDF · 20 pages](docs/publications/vons-tech-report-v1.0.1.pdf) | + +Articles and original figures use **CC BY 4.0**. Their version, SHA-256 digests +and verification scope are in the [publication index](docs/publications/README.md). +These preprints preserve negative results and missing measurements. No arXiv +identifier or peer-review acceptance is announced. + +## Local use + +```sh +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -e '.[dev]' +python -m vons.cli generate-smoke --output data/generated/smoke.jsonl +python -m vons.cli validate-data --input data/generated/smoke.jsonl +``` + +The source export includes the research tools and pinned pilot configurations. +Obtain and review upstream assets separately before running model experiments. +Raw restricted benchmark data and private evidence are not redistributed here. + +## Terms + +Source is distributed under [Vons Community and Commercial License 1.0](LICENSE): +qualifying noncommercial use is free; enterprise and other commercial use require +a separate written agreement. Contact oswarld@inlevel9.com for commercial terms. +This is source-available software, not OSI-approved open source. + +The paper and Tech Report are distributed under [CC BY 4.0](docs/publications/LICENSE.md). +This permits commercial article reuse with attribution; software rights are +separate. Third-party assets retain their own terms. See +[licensing scope](docs/LICENSING.md). This source release does not establish research validation or a commercial +customer agreement. diff --git a/configs/pilot-diffusion.json b/configs/pilot-diffusion.json new file mode 100644 index 0000000000000000000000000000000000000000..b5a2315bef1370f51b68fcc8b3bb95c03ca0e6e0 --- /dev/null +++ b/configs/pilot-diffusion.json @@ -0,0 +1,16 @@ +{ + "model_id": "vons-pilot-diffusion", + "backend": "diffusion", + "option_count": 32, + "encoder": {"name": "google/bert_uncased_L-4_H-256_A-4", "revision": "387825ce42dbb39b87911cdf8e383ee3b25184f8", "max_tokens": 512}, + "data": {"path": "data/generated/pilot.jsonl", "manifest": "artifacts/pilot-diffusion/data-manifest.json"}, + "seed": 7, + "epochs": 3, + "batch_size": 16, + "learning_rate": 0.00005, + "diffusion_steps": 32, + "inference_steps": 4, + "device": "auto", + "artifact_dir": "artifacts/pilot-diffusion", + "target_bundle_bytes": 67108864 +} diff --git a/configs/pilot.json b/configs/pilot.json new file mode 100644 index 0000000000000000000000000000000000000000..b289157644c0014c391c993997546de5c9090317 --- /dev/null +++ b/configs/pilot.json @@ -0,0 +1,15 @@ +{ + "model_id": "vons-pilot-direct", + "backend": "direct", + "option_count": 32, + "encoder": {"name": "google/bert_uncased_L-4_H-256_A-4", "revision": "387825ce42dbb39b87911cdf8e383ee3b25184f8", "max_tokens": 512}, + "data": {"path": "data/generated/pilot.jsonl", "manifest": "artifacts/pilot/data-manifest.json"}, + "seed": 7, + "epochs": 3, + "batch_size": 16, + "learning_rate": 0.00005, + "diffusion_steps": 32, + "inference_steps": 4, + "device": "auto", + "target_bundle_bytes": 67108864 +} diff --git a/configs/public-release.json b/configs/public-release.json new file mode 100644 index 0000000000000000000000000000000000000000..77838a70b756c0abfc8a044a16d56d3e69be8d65 --- /dev/null +++ b/configs/public-release.json @@ -0,0 +1,111 @@ +{ + "schema": "vons.public-source-spec/v1", + "license_status": "owner_selected_custom_terms_prepared", + "files": [ + ".gitattributes", + ".github/ISSUE_TEMPLATE/bug.yml", + ".github/ISSUE_TEMPLATE/experience.yml", + ".github/ISSUE_TEMPLATE/reproduction.yml", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/workflows/checks.yml", + ".gitignore", + "CITATION.cff", + "CONTRIBUTING.md", + "LICENSE", + "README.md", + "configs/pilot-diffusion.json", + "configs/pilot.json", + "configs/public-release.json", + "docs/COMMUNITY.md", + "docs/HUB_PUBLISHING.md", + "docs/HUGGING_FACE.md", + "docs/KASI_INTEGRATION.md", + "docs/LICENSING.md", + "docs/MODEL_CARD.md", + "docs/PUBLIC_RELEASE.md", + "docs/assets/inlevel9-signature-dark.png", + "docs/assets/inlevel9-signature.png", + "docs/publications/ARXIV.md", + "docs/publications/LICENSE.md", + "docs/publications/README.md", + "docs/publications/vons-paper-v1.0.pdf", + "docs/publications/vons-tech-report-v1.0.1.pdf", + "pyproject.toml", + "sdk/typescript/LICENSE", + "sdk/typescript/README.md", + "sdk/typescript/demo/benchmark.html", + "sdk/typescript/demo/benchmark.ts", + "sdk/typescript/demo/index.html", + "sdk/typescript/demo/main.ts", + "sdk/typescript/demo/parity.html", + "sdk/typescript/demo/parity.ts", + "sdk/typescript/package-lock.json", + "sdk/typescript/package.json", + "sdk/typescript/src/index.ts", + "sdk/typescript/src/onnx.ts", + "sdk/typescript/test/index.test.mjs", + "sdk/typescript/test/onnx.test.mjs", + "sdk/typescript/test/parity.test.mjs", + "tests/test_browser_benchmark.py", + "tests/test_browser_benchmark_summary.py", + "tests/test_browser_parity_reference.py", + "tests/test_bundle.py", + "tests/test_calibration.py", + "tests/test_contract.py", + "tests/test_data.py", + "tests/test_evaluation.py", + "tests/test_merge_browser_benchmarks.py", + "tests/test_mind2web.py", + "tests/test_mind2web_import.py", + "tests/test_mind2web_manifest.py", + "tests/test_mind2web_prediction.py", + "tests/test_mind2web_replay.py", + "tests/test_mind2web_runner.py", + "tests/test_public_release.py", + "tests/test_quality_summary.py", + "tests/test_regression.py", + "tests/test_shape_benchmark.py", + "tests/test_shared_bundle.py", + "tools/archive_manifest.py", + "tools/audit_local_model_alignment.py", + "tools/benchmark_onnx_bundle.py", + "tools/benchmark_onnx_shapes.py", + "tools/build_claim_ledger.py", + "tools/build_mind2web_manifest.py", + "tools/evaluate_mind2web.py", + "tools/export_shared_bundle.py", + "tools/freeze_publication_packet.py", + "tools/import_mind2web.py", + "tools/merge_browser_benchmarks.py", + "tools/predict_mind2web.py", + "tools/prepare_public_release.py", + "tools/recover_historical_manifest.py", + "tools/replay_mind2web_evaluation.py", + "tools/summarize_browser_benchmark.py", + "tools/summarize_quality.py", + "tools/verify_browser_benchmark.py", + "tools/verify_browser_parity.py", + "tools/verify_onnx_bundle.py", + "vons/__init__.py", + "vons/bundle.py", + "vons/calibration.py", + "vons/cli.py", + "vons/contract.py", + "vons/data.py", + "vons/evaluation.py", + "vons/kasi.py", + "vons/mind2web.py", + "vons/models.py", + "vons/ollama.py", + "vons/research.py", + "vons/training.py" + ], + "reviewed_binaries": { + "docs/assets/inlevel9-signature.png": "f7514b56b2a93293054b3d22634ac5f0cb11e00a370f53d8d9f934c55a21bbb5", + "docs/assets/inlevel9-signature-dark.png": "d6645528158842af77a89de97361fa0c521fbc5a3c0ea6e996f9c912cfcfe9b5", + "docs/publications/vons-paper-v1.0.pdf": "ae0e09c60e17a98b52bb25c611b2489d10b18bd98d87974f2e8dd25d746a5fc8", + "docs/publications/vons-tech-report-v1.0.1.pdf": "33076d9e52d4a073060066215dd390e3b0bf0cdef48c5ffb2a7c5189854254ed" + }, + "software_license": "LicenseRef-Vons-Community-Commercial-1.0", + "article_license": "CC-BY-4.0" +} diff --git a/docs/COMMUNITY.md b/docs/COMMUNITY.md new file mode 100644 index 0000000000000000000000000000000000000000..d2e683b107d9b9d351045d20fc9b38232932a2cb --- /dev/null +++ b/docs/COMMUNITY.md @@ -0,0 +1,115 @@ +# Read, try, share and contribute + +Vons explores compact local decisions inside agent workflows. A planner supplies +state and bounded candidates; the host retains execution, policy and consent. +The community should help people understand this boundary, try the software and +report what works or fails. + +## Start here + +| Your goal | Entry point | Current availability | +| --- | --- | --- | +| Understand Vons | [README](../README.md) and [model card](MODEL_CARD.md) | Included in this source preview | +| Read the research | [Paper and Tech Report](publications/README.md) | Coming soon; v1.0 manuscripts are in preparation and excluded from this release | +| Try without weights | The walkthrough below | Deterministic contract and synthetic data only | +| Run model inference | [TypeScript SDK](../sdk/typescript/README.md) | Requires separately obtained, reviewed model/tokenizer assets | +| Share experience | [Experience report](https://github.com/inlevel9-com/Vons/issues/new?template=experience.yml) | GitHub account required to submit | +| Reproduce a result | [Reproduction report](https://github.com/inlevel9-com/Vons/issues/new?template=reproduction.yml) | Form included; evidence reviewed separately | +| Improve Vons | [Contributing](../CONTRIBUTING.md) | Documentation, tests and scoped code changes welcome | + +The destinations are [GitHub: inlevel9-com/Vons](https://github.com/inlevel9-com/Vons) +and [Hugging Face: INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons). +The source preview includes documentation and GitHub issue forms for feedback. +The paper and Tech Report will be added after v1.0 review; this release contains +neither manuscript. A source upload does not provide hosted model inference. +The current software terms require a separate agreement for +enterprise/commercial use, including enterprise evaluation. Reading the paper +under its article terms is separate from software use. + +## A five-minute first exploration + +Five minutes is a usability target, not a measured onboarding result. Start in +the root of a source checkout with Python 3.10+. This path needs no model assets, +GPU, API token or hosted inference account. Python packaging may need internet +access to install its build tools. + +```sh +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -e . +python -m vons.cli generate-smoke --output data/generated/community-smoke.jsonl +python -m vons.cli validate-data --input data/generated/community-smoke.jsonl +``` + +On Windows PowerShell use `.venv\Scripts\Activate.ps1` for activation. The +generator reports four synthetic rows. Validation prints their data manifest; +this checks the data contract, not model quality. + +Run this Python example in the same environment: + +```python +from vons import KASIAdapter, KASIProposal + +adapter = KASIAdapter(tool_policy={"read_weather": "low"}) +for tool in ("read_weather", "unknown_tool"): + proposal = KASIProposal.from_mapping({ + "calls": [{"name": tool}], + "confidence": 0.9, + "risk": "low", + }) + decision = adapter.decide(proposal) + print(tool, decision.action.value, decision.reason) +``` + +Expected output: + +```text +read_weather call approved_by_host_policy +unknown_tool refuse unregistered_or_no_proposed_call +``` + +This is the deterministic KASI adapter, not a learned Vons prediction. The word +`call` is a returned decision; the example never fetches weather or executes a +tool. The host supplies the tool policy. The literal confidence is sample input, +not a calibrated probability. See the [general scope](MODEL_CARD.md) and +[KASI integration](KASI_INTEGRATION.md) before building an integration. + +## Share an honest experience + +Use a report form with the source revision or release-manifest digest, scenario, +OS/device, runtime/provider, exact command, expected behavior and observed +behavior. Mark whether you tried the contract, a synthetic example or real model +inference. Include attempted, successful, abstained and failed counts with +definitions; explain any overlapping categories. Report unavailable quantities +as unavailable and keep errors in the denominator. + +A short negative report is useful. Personal experience and measured benchmarks +serve different purposes: user ratings, stars and downloads do not establish +accuracy, safety or independent replication. Public reports are not automatically +training data or promotional testimonials. Link only safe evidence you have +reviewed and have permission to share. No name, employer or private workflow is +required beyond the hosting service's account requirements. + +## A practical platform roadmap + +These are proposed acceptance criteria, not completed or measured milestones. + +1. **Reading and participation:** publish a reviewed source preview, readable + research summary and accurate availability notices. A signed-out visitor can + read the documentation; a signed-in contributor can submit each issue form + and find the contribution rules. Check keyboard navigation and narrow screens. +2. **First use:** test the no-weight walkthrough from a clean checkout. Invite + voluntary reports of completion, failure and time spent. Keep counts and + denominators; do not infer actual use from page views or stars. +3. **Real inference:** offer a versioned demo only after permitted model assets, + tokenizer, notices, provider evidence and numerical parity are reviewed. + Clearly label unsupported requests and known failures. Host policy still + controls actions. Do not present static or simulated results as inference. +4. **Community learning:** triage reports, publish reproducible improvements, + credit contributions and keep corrections visible. Record submitted reports, + independently reproduced reports and accepted contributions separately. + +Start with the existing GitHub and Hugging Face channels. A dedicated website, +review database, accounts and telemetry need a separate implementation decision; +none is required for the first source preview. Reading should remain available +without a Vons account. Follow the [contribution and moderation rules](../CONTRIBUTING.md). diff --git a/docs/HUB_PUBLISHING.md b/docs/HUB_PUBLISHING.md new file mode 100644 index 0000000000000000000000000000000000000000..2ac3ca698274eac34b81ac05b6e40b2af89da632 --- /dev/null +++ b/docs/HUB_PUBLISHING.md @@ -0,0 +1,118 @@ +# Upload the Vons source preview to Hugging Face + +This is an owner-operated upload guide. The current local preparation contains +97 reviewed files, including the final paper and Tech Report PDFs. Drafts, +source manuscript bundles, model weights, restricted data and private records +are excluded. + +## Destination and scope + +Use the existing model repository [INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons). +The organization page [INLEVEL9](https://huggingface.co/INLEVEL9) is its parent, +not a folder-upload destination. There is no need to create another repository, +a dataset or a Space for this source preview. + +At the last read-only check on 2026-09-25, the Hub repository was public and +contained only `.gitattributes` and a minimal Apache-2.0 README. The prepared +root README and LICENSE replace that placeholder with the selected Vons +Community and Commercial License 1.0. The article CC BY 4.0 terms are separate. + +A commit to this public repository makes uploaded files publicly readable. +Uploading source does not create hosted inference, pretrained weights or a +working `from_pretrained` package. The card states those limits explicitly. +The main code/contribution destination is +[inlevel9-com/Vons](https://github.com/inlevel9-com/Vons). Its visibility is +controlled separately from Hugging Face. + +## 1. Install and sign in + +On macOS with Homebrew: + +```sh +brew install hf +hf auth login +hf auth whoami +``` + +If Homebrew already installed `hf`, use `brew upgrade hf` when an update is +needed. Current CLI versions offer browser login: follow the displayed URL and +code, and sign in using your account with write access to `INLEVEL9/Vons`. +`hf auth whoami` confirms the account and organization membership; membership +alone does not guarantee repository write permission. + +If your CLI asks for a token instead, create a user access token in +[Hugging Face token settings](https://huggingface.co/settings/tokens), preferably +scoped to write to `INLEVEL9/Vons`, and enter it at the hidden login prompt. +Keep credentials out of source files, shell command arguments and issue reports. +See the official [CLI guide](https://huggingface.co/docs/huggingface_hub/guides/cli). + +## 2. Use the reviewed export + +Run the source audit from the Vons checkout: + +```sh +python3 tools/prepare_public_release.py +``` + +The current prepared upload folder is `releases/huggingface-source-v8`. +It contains the 97 reviewed files plus a local `RELEASE_MANIFEST.json` receipt +with every file's byte count and SHA-256 digest. Its root README is the Hub model +card; the working-tree README remains the GitHub landing page. + +This export is a snapshot. If you edit any public source or documentation after +preparation, create a fresh versioned directory and use that path below: + +```sh +python3 tools/prepare_public_release.py --target huggingface --output releases/huggingface-source-v9 +``` + +The exporter refuses to overwrite an existing directory. Never point the upload +command at the development checkout or an old export containing stale terms. + +## 3. Upload + +From the Vons checkout, after reviewing the v8 export: + +```sh +hf upload INLEVEL9/Vons releases/huggingface-source-v8 . \ + --repo-type model \ + --exclude RELEASE_MANIFEST.json \ + --commit-message "Publish Vons source preview and reviewed research PDFs" +``` + +The final `.` places the files at the repository root while preserving nested +paths. The command creates a Hub commit directly; a separate Git commit/push +is unnecessary. It updates matching remote paths, including README and LICENSE, +and does not request deletion of unrelated remote files. The manifest stays +local as an audit receipt; it is not an additional source file in the allowlist. + +A successful command prints a repository or commit URL. For a 401/403 response, +check the account, `INLEVEL9/Vons` write permission and credential scope; do not +create a duplicate personal repository. Official +[upload guidance](https://huggingface.co/docs/huggingface_hub/guides/upload) +describes folder uploads and filtering. + +## 4. Verify the result + +Open [the model card](https://huggingface.co/INLEVEL9/Vons) and +[Files and versions](https://huggingface.co/INLEVEL9/Vons/tree/main): + +- The card says this is a source preview with no pretrained weights or hosted inference. +- License metadata uses `other`, `vons-community-commercial-1.0` and `LICENSE`. + It must no longer identify the software as Apache-2.0 or CC BY 4.0. +- Root `LICENSE` and `sdk/typescript/LICENSE` contain the same custom terms. +- Source paths such as `vons/`, `tools/`, `tests/` and `sdk/typescript/` are present. +- The Paper and Tech Report links open the 15-page paper and 20-page report. + Their digests match the publication index. No manuscript source bundle or + private review material is uploaded. +- Logo/documentation links resolve, and a signed-out visitor can read the card. + +Save the resulting Hub commit URL with the local release manifest. GitHub +publication, manuscript publication and a hosted demo are separate actions. +For subsequent source changes, create a new reviewed export and repeat the same +upload command with its new path. Add research files only after their separate +v1.0 review and an explicit public-allowlist update. + +The owner performs authentication, commit, push and publication. This local +preparation has not executed an upload or changed remote visibility. +Official CLI/upload guidance checked 2026-09-25. diff --git a/docs/HUGGING_FACE.md b/docs/HUGGING_FACE.md new file mode 100644 index 0000000000000000000000000000000000000000..770bf1395c322c08a80f68b135915045258783fd --- /dev/null +++ b/docs/HUGGING_FACE.md @@ -0,0 +1,93 @@ +--- +license: other +license_name: vons-community-commercial-1.0 +license_link: LICENSE +language: + - en +tags: + - vons + - research + - candidate-selection + - onnx +--- + +![INLEVEL9](../docs/assets/inlevel9-signature.png) + +# Vons + +**Plan globally. Decide locally. Keep the host in control.** + +Compact local decision models for frontier-agent workflows. + +**Kwangseob Ahn** · **INLEVEL9 / SEJONG UNIV.** · **oswarld@inlevel9.com** + +## Source preview + +This repository contains research source and documentation. It does not contain +pretrained model weights, tokenizer assets or datasets. Hosted inference and +`from_pretrained` loading are not available from this source preview. + +Vons studies Direct and Diffusion candidate scoring with abstention. The host +owns execution, permission and consent. Model output never authorizes an action. +English-first operation and a model asset budget under 64 MiB are design targets; +they are not general quality or complete browser-package guarantees. + +See [the model card](../docs/MODEL_CARD.md), +[the TypeScript SDK](../sdk/typescript/README.md), +[the paper and Tech Report](../docs/publications/README.md), and +[release preparation](../docs/PUBLIC_RELEASE.md). + +## Read, try and contribute + +This source preview is distributed through [INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons). +Source and contribution reports belong in +[inlevel9-com/Vons](https://github.com/inlevel9-com/Vons). The reviewed paper and Tech Report are included as PDF downloads below. The software remains a 0.1.0 +research preview. No pretrained weights or working Space are announced. + +Follow the [community guide](../docs/COMMUNITY.md) for the no-weight walkthrough, +experience and reproduction reports, and the +[contribution guide](../CONTRIBUTING.md) for review expectations. The paper is +being prepared for [arXiv](../docs/publications/ARXIV.md). User reports are not +independent research validation. + +Maintainers can use the [Hub upload guide](../docs/HUB_PUBLISHING.md) to prepare +an organization card, source preview and later a separately validated Space. + +## Paper and Tech Report + +| Publication | Download | +| --- | --- | +| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows — paper v1.0, final editorial revision | [PDF · 15 pages](../docs/publications/vons-paper-v1.0.pdf) | +| Vons: Compact Decision Models for Frontier-Agent Workflows — Technical Report v1.0.1 | [PDF · 20 pages](../docs/publications/vons-tech-report-v1.0.1.pdf) | + +Articles and original figures use **CC BY 4.0**. Their version, SHA-256 digests +and verification scope are in the [publication index](../docs/publications/README.md). +These preprints preserve negative results and missing measurements. No arXiv +identifier or peer-review acceptance is announced. + +## Local use + +```sh +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -e '.[dev]' +python -m vons.cli generate-smoke --output data/generated/smoke.jsonl +python -m vons.cli validate-data --input data/generated/smoke.jsonl +``` + +The source export includes the research tools and pinned pilot configurations. +Obtain and review upstream assets separately before running model experiments. +Raw restricted benchmark data and private evidence are not redistributed here. + +## Terms + +Source is distributed under [Vons Community and Commercial License 1.0](../LICENSE): +qualifying noncommercial use is free; enterprise and other commercial use require +a separate written agreement. Contact oswarld@inlevel9.com for commercial terms. +This is source-available software, not OSI-approved open source. + +The paper and Tech Report are distributed under [CC BY 4.0](../docs/publications/LICENSE.md). +This permits commercial article reuse with attribution; software rights are +separate. Third-party assets retain their own terms. See +[licensing scope](../docs/LICENSING.md). This source release does not establish research validation or a commercial +customer agreement. diff --git a/docs/KASI_INTEGRATION.md b/docs/KASI_INTEGRATION.md new file mode 100644 index 0000000000000000000000000000000000000000..9c5b12ade57c8a7b5a1b33fe6605ba47ebb5b531 --- /dev/null +++ b/docs/KASI_INTEGRATION.md @@ -0,0 +1,44 @@ +# KASI integration in Vons + +KASI is the first Vons domain adapter, not a separate model product. Vons owns +the general candidate judgment contract; the adapter preserves KASI's public +five-action contract for tool-calling hosts: + +`call`, `clarify`, `confirm`, `refuse`, `respond`. + +The model may propose ordinary calls, clarification, refusal, or a response. It +cannot grant consent. The host wrapper requires a host-owned registry of tool +names and risk levels, validates tools and arguments, applies confidence +thresholds, and derives `confirm` from that policy. The proposal's `risk` is +only advisory: it can never lower the registered host risk, and missing or +unknown risk is fail-closed. The host remains the only execution boundary. + +Consent is bound to the complete call, including its arguments. Pass the +`KASICall.fingerprint()` returned for the exact call in `confirmed_calls`; a +tool name in `confirmed_tools` is retained only for compatibility and cannot +authorize a high-risk call. + +The Python adapter stores a SHA-256 fingerprint. The TypeScript SDK exposes the +same canonical `{name, arguments}` consent key as a portable string because +browser runtimes do not provide synchronous SHA-256. A host that shares consent +records between the Python and TypeScript adapters must hash the TypeScript key +with SHA-256 before persistence and comparison. + +Unregistered tools are refused. A minimal host setup is: + +```python +from vons import KASIAdapter, KASICall, KASIProposal + +adapter = KASIAdapter(tool_policy={"read_weather": "low", "unlock": "high"}) +proposal = KASIProposal( + calls=(KASICall("read_weather", {"city": "Seoul"}),), + confidence=0.9, + risk="low", +) +decision = adapter.decide(proposal) +``` + +The mapping is intentionally one-way: KASI calls can be represented as Vons +choices, but arbitrary Vons choices do not become tool calls without a host +adapter. Existing KASI datasets, safety policy, and C99 adapter can be imported +behind a manifest when their provenance and licensing are recorded. diff --git a/docs/LICENSING.md b/docs/LICENSING.md new file mode 100644 index 0000000000000000000000000000000000000000..ae2f2d45af21f5dafde4999fe1d103e85f373564 --- /dev/null +++ b/docs/LICENSING.md @@ -0,0 +1,42 @@ +# Licensing and distribution scope + +The owner selected these terms on 2026-09-24. This local preparation is ready +for owner-controlled source distribution: + +| Material | Terms | +|---|---| +| Vons-owned source, tests, configurations and software documentation | [Vons Community and Commercial License 1.0](../LICENSE) | +| Paper, Tech Report and their original research figures | [CC BY 4.0](publications/LICENSE.md) | +| Third-party libraries, upstream encoders/tokenizers and datasets | Their original terms; no relicense | +| INLEVEL9 logo and other marks | No general trademark license or endorsement | +| Model weights, restricted data and private evidence | Excluded from this source release | + +The free software grant covers qualifying noncommercial personal, educational, +nonprofit research and public-benefit use. Enterprise use, including internal +evaluation, and other commercial use require a separate written agreement. +Paid consulting, client work, SaaS/API hosting and embedding in a commercial +product fall in that category. Contact [oswarld@inlevel9.com](mailto:oswarld@inlevel9.com) +for the scope and commercial terms; no price or automatic trial is specified. + +This is **source-available software**, not OSI-approved open source. The article +license is deliberately separate: CC BY 4.0 permits commercial article reuse +with its attribution conditions, without granting commercial software rights. +Affiliation is attribution and does not establish institutional endorsement. + +## Transition and distribution + +Earlier local Python metadata declared Apache-2.0. This source release +uses `LicenseRef-Vons-Community-Commercial-1.0` and includes its exact text. +Frozen research snapshots retain their historical metadata and measurements. +This change makes no claim that rights previously granted under Apache-2.0 can +be withdrawn. Existing third-party licenses and notices remain effective. + +The root license contains the custom terms for this source distribution. +A separate written agreement is required for enterprise or commercial software +use; publishing the source does not execute a customer contract. The source +list and hashes identify the reviewed files, and do not establish rights over +an unreviewed third-party asset. No manuscript or model weights accompany this +source release. + +Commercial contracts, pricing, support and optional conformance services are +separate from this free community grant. No certification claim is made. diff --git a/docs/MODEL_CARD.md b/docs/MODEL_CARD.md new file mode 100644 index 0000000000000000000000000000000000000000..e6baacc93897fa44fe10448b0946228be7a02d1e --- /dev/null +++ b/docs/MODEL_CARD.md @@ -0,0 +1,54 @@ +# Vons model card + +Status: research source preview. This package contains no pretrained weights, +tokenizer files or benchmark datasets. + +## Intended use + +Vons studies bounded candidate selection and abstention in agent workflows. +A host supplies the state and candidates. The host alone enforces permission, +consent and execution policy. KASI is a separate compatibility adapter. + +## Architecture and target + +The research implementation offers a Direct scorer and a conditional Diffusion +scorer. Pilot configurations pin `google/bert_uncased_L-4_H-256_A-4` at revision +`387825ce42dbb39b87911cdf8e383ee3b25184f8`. This identifies a research dependency; +its assets are not included or relicensed here. + +English-first input and a model asset budget below 64 MiB are design targets. +Runtime libraries, complete download size and memory use are distinct quantities. + +## Evaluation and limitations + +Historical local experiments include synthetic pilots and an evaluation-only +external benchmark diagnostic. They do not establish general task quality, +production readiness, calibrated safety or a Direct/Diffusion superiority claim. +Reviewed numerical claims belong to the versioned +[paper and Tech Report](publications/README.md), with exact evidence references. +Unreviewed historical tables are not distributed as current results. + +The ONNX candidate head does not support general score questions. Python and +TypeScript token-budget checks are not fully identical. Browser smoke, repeated +latency, full numerical parity and memory evidence remain separate gates. + +Do not use model confidence as a permission grant or assume it is calibrated. +Vons cannot execute tools or approve high-risk actions. + +## Data and provenance + +Synthetic data can be generated from the source. Restricted data, model-generated +raw responses, private prompts, model checkpoints and local manifests are +excluded from this distribution. Obtain any external dataset under its own +terms. The source release manifest records hashes of the files actually exported. + +## Author and terms + +Kwangseob Ahn — INLEVEL9 / SEJONG UNIV. — oswarld@inlevel9.com. + +The Vons-owned source is prepared under [Vons Community and Commercial License +1.0](../LICENSE): noncommercial community use is free; enterprise/commercial use +requires a separate written agreement. The articles and their original figures +use [CC BY 4.0](publications/LICENSE.md). Third-party assets retain their own +terms, and no model weights are licensed or included by this source-only card. +See [licensing scope](LICENSING.md) and [release preparation](PUBLIC_RELEASE.md). diff --git a/docs/PUBLIC_RELEASE.md b/docs/PUBLIC_RELEASE.md new file mode 100644 index 0000000000000000000000000000000000000000..4937ffa55bdeea9746029f69d4c382df51a50094 --- /dev/null +++ b/docs/PUBLIC_RELEASE.md @@ -0,0 +1,94 @@ +# Public source release + +The public source set is defined by `configs/public-release.json`. The root +`.gitignore` excludes unreviewed root paths and internal documents; the release +tool independently rejects any Git candidate outside that exact file list. + +## Included and excluded material + +Included: Python and TypeScript source, tests, source demo files, the SDK +dependency lockfile, pilot configuration, reviewed public documentation and the +owner-supplied INLEVEL9 logo. Reviewed publications are listed separately in +`docs/publications/README.md`. + +Excluded: all local datasets (including synthetic outputs), model weights, +tokenizers, ONNX external-data shards, experiment reports, evidence archives, +private prompts and raw model responses, agent/session records, license drafts, +historical planning documents, generated demo JavaScript, environments, +credentials, service caches and unreviewed binary files. Keep excluded originals +locally; never force-add them to make an upload work. + +## Audit and export + +Run from the repository root with Python 3.10+ and Git: + +```sh +python tools/prepare_public_release.py +python tools/prepare_public_release.py --target github --output releases/github-source +python tools/prepare_public_release.py --target huggingface --output releases/huggingface-source +``` + +Each export is a new directory containing only allowlisted files and a +`RELEASE_MANIFEST.json` with byte counts and SHA-256 hashes. Existing output +directories are never overwritten. Use a new versioned output path after any +source change. The Hugging Face export substitutes its model card as the root +README. Both exports contain source and the two exact reviewed publication PDFs, with +no pretrained assets. + +The audit detects known token/key patterns, personal home paths, private writing +session URLs, unexpected Git files, symlinks and changed reviewed binaries. +Pattern scanning is heuristic: passing it is not a complete secret audit or +proof of redistribution rights. Read the actual export before publishing. + +## GitHub + +The source checkout uses `main`; review `git status`, the existing commit history +and the export manifest before publishing changes. The owner performs commits, +pushes and visibility changes. This preparation does not perform those actions. Preserve a clean initial tree: `.gitignore` cannot +remove already committed material from history. If existing remote history is +later attached, review that history independently before making it public. + +The CI workflow checks the release allowlist, Python tests and lint, and the SDK +tests and three browser builds. Push/pull-request checks run when source, +tests, configuration or the workflow changes; documentation-only edits do not +trigger the full suite. A manual run remains available. SDK tokenizer tests use +synthetic fixtures and require no private pilot assets. It has read-only repository permissions and +does not publish packages or artifacts. Checks that require unavailable optional +Python dependencies may be skipped; install training/export extras for the full +local suite. The historical pilot-summary regression is also skipped when its +private input files are absent. A separate synthetic regression runs without +those files; do not publish the historical inputs merely to remove that skip. + +## Hugging Face + +Upload only a reviewed versioned Hugging Face export to `INLEVEL9/Vons`. +The current preparation is `releases/huggingface-source-v8`; see the +[step-by-step upload guide](HUB_PUBLISHING.md). Do not select the development workspace in the +Hub upload dialog. Use explicit allow/ignore patterns if using `upload_folder`; +its handling of ignore files depends on the client version and upload method. + +This source preview has no weights and no inference widget. A later weight +release needs its own exact file inventory, upstream notices, model/data +provenance and runtime validation. Do not turn the ignored checkpoint directory +into a public upload by adding Git LFS patterns. + +Official references: [GitHub ignore guidance](https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files) +and [Hub upload guidance](https://huggingface.co/docs/huggingface_hub/guides/upload). + +## Terms and publication state + +The owner selected noncommercial-free / enterprise-commercial software terms +and CC BY 4.0 articles on 2026-09-24. The prepared root [license](../LICENSE), SDK +license copy, package metadata and Hub card now match that source-available +direction. [Licensing scope](LICENSING.md) explains the boundary and preserves +earlier and third-party rights. Local preparation is complete for the owner to +commit, push and publish; no commercial customer agreement is adopted here. +npm publishing remains disabled with `private: true`. + +The Python build backend requires setuptools 77.0.3+ to include the custom SPDX +license reference and exact license file in wheel/sdist metadata. No runtime +dependency was added. Verify built package contents as well as source exports. + +The author-approved public contact is Kwangseob Ahn, oswarld@inlevel9.com, +INLEVEL9 / SEJONG UNIV. Logo inclusion does not grant trademark rights or imply +institutional endorsement. Model-assisted writing/review is not peer acceptance. diff --git a/docs/assets/inlevel9-signature-dark.png b/docs/assets/inlevel9-signature-dark.png new file mode 100644 index 0000000000000000000000000000000000000000..431a41050572a8b47354ab2358a1093f247ee0ab Binary files /dev/null and b/docs/assets/inlevel9-signature-dark.png differ diff --git a/docs/assets/inlevel9-signature.png b/docs/assets/inlevel9-signature.png new file mode 100644 index 0000000000000000000000000000000000000000..2bb449af8364fc25437590fd76c8c4a37952d916 Binary files /dev/null and b/docs/assets/inlevel9-signature.png differ diff --git a/docs/publications/ARXIV.md b/docs/publications/ARXIV.md new file mode 100644 index 0000000000000000000000000000000000000000..9ce87be444d5a204e56b02e6eea53d1e35011dc8 --- /dev/null +++ b/docs/publications/ARXIV.md @@ -0,0 +1,67 @@ +# arXiv preparation + +The [reviewed paper PDF](vons-paper-v1.0.pdf) is included in the source release, +alongside the [Technical Report](vons-tech-report-v1.0.1.pdf). arXiv submission +is being prepared; no arXiv identifier or acceptance is announced. arXiv performs +content moderation, not journal peer review; see its +[moderation policy](https://info.arxiv.org/help/moderation/index.html). + +## Current preparation + +Exact local PDF hashes match the reviewed final manuscripts, and all 35 pages +have been inspected. The documents were generated with Typst, not TeX/LaTeX. +The paper is being prepared for the genuine non-TeX PDF submission route, +subject to arXiv's processing checks. An independent source rebuild remains +unverified; see the [publication index](README.md) for exact verification scope. +Historical drafts and claim-to-source records remain private. + +## Research scope + +Present the bounded decision contract, engineering evidence and informative +negative results. State the synthetic pilot's label imbalance and baseline, +external-evaluation failures, token budget and browser-verification limitations +alongside the relevant results. A narrower research preprint can describe open +engineering gates without claiming that those gates have passed or that a model +is ready for deployment. Preserve a separate private claim-to-source audit. + +The publication language is English. Author metadata is **Kwangseob Ahn**, +**INLEVEL9 / SEJONG UNIV.**, **oswarld@inlevel9.com**. Article preparation uses +[CC BY 4.0](LICENSE.md); software terms are separate. The author remains +responsible for scientific claims, references and model-assistance disclosure. + +## Files and checks + +Prepare a readable PDF, editable manuscript source, included figures and +bibliography, together with a reproducible build instruction. Verify every +source-bundle entry and record exact hashes. Resolve missing references, +orphan captions, table breaks and unavailable figures; inspect all PDF pages. +Keep private claim ledgers, prompts, review conversations, restricted benchmark +rows and unrelated repository files outside the submission bundle. + +arXiv prefers TeX/LaTeX and requires source when the PDF was made from TeX/LaTeX. +Include figure files instead of external figure links. See the official +[submission overview](https://info.arxiv.org/help/submit/index.html). +For a genuine non-TeX PDF, follow the [PDF requirements](https://info.arxiv.org/help/submit_pdf.html), +including embedded outline fonts and no encryption or active content. +For a TeX package, include the necessary bibliography source or compatible +generated bibliography and confirm the top-level file builds from the bundle +root. Inspect arXiv's processed PDF as well as the local build. See +[TeX submission guidance](https://info.arxiv.org/help/submit_tex.html). + +## Author submission handoff + +1. Review the exact final manuscript, source bundle, hashes and remaining limits. +2. Confirm title, abstract, author affiliation, article rights and a subject + category based on the paper's actual contribution. `cs.AI` and `cs.LG` are + candidates to assess, not confirmed classifications. +3. Check the account and any applicable [endorsement requirement](https://info.arxiv.org/help/endorsement.html). +4. Prepare the upload and metadata; verify the processed PDF and the selected + [arXiv license](https://info.arxiv.org/help/license/index.html). CC BY 4.0 is + an available option consistent with the selected article terms. +5. The author approves the concrete submission and completes the submission + agreement. Add the real arXiv URL to the publication index only after it exists. + +Official guidance was checked on 2026-09-25; recheck it at submission time. +The GitHub destination is [inlevel9-com/Vons](https://github.com/inlevel9-com/Vons) +and the Hugging Face repository is [INLEVEL9/Vons](https://huggingface.co/INLEVEL9/Vons). +Neither link establishes public artifact availability, replication or acceptance. diff --git a/docs/publications/LICENSE.md b/docs/publications/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..6ca2be0c3cea32638262661f9bc8766429d51f6c --- /dev/null +++ b/docs/publications/LICENSE.md @@ -0,0 +1,40 @@ +# Article license: CC BY 4.0 + +The Vons paper and Tech Report, including their original research figures, are +prepared for release under the **Creative Commons Attribution 4.0 International +License (CC BY 4.0)**, as selected by the author on 2026-09-24. + +Author: Kwangseob Ahn +Affiliation: INLEVEL9 / SEJONG UNIV. +Contact: oswarld@inlevel9.com + +The [CC BY 4.0 legal code](https://creativecommons.org/licenses/by/4.0/legalcode.en) +governs the article material identified by this notice. The +[license summary](https://creativecommons.org/licenses/by/4.0/) explains sharing, +adaptation and attribution. Commercial reuse of the articles is permitted under +CC BY 4.0; the software's commercial-use restriction does not apply to them. + +Retain the author, title, version, license link and supplied attribution notices; +indicate modifications. Do not imply the author's or institutions' endorsement. +Third-party material specifically identified under other terms, software source +excerpts identified as separately licensed, logos, raw benchmark data and private +verification records are excluded from this article grant. Statutory exceptions +and independent rights remain unaffected. + +This notice does not license the Vons software or any model weights. See the +repository's [software license](../../LICENSE) for software use. It does not +claim that an article has been submitted, accepted, externally peer-reviewed or +already uploaded to a public destination. + +## Notice for the v0.2 release copies + +This notice records the author's decision after the research evidence was +frozen. It applies to the selected release copies of the Vons paper v0.2 and +Technical Report v0.2 and their original figures. Where an earlier manuscript +states that article terms await an owner decision, this notice supersedes that +article-license status only. It does not alter measurements, claim provenance, +publication status, or the remaining research gates. + +Distribute this notice alongside the reviewed manuscript files and their +release checksums. The Tech Report's original author/model attribution is +preserved; a separate licensing notice does not replace its authoring history. diff --git a/docs/publications/README.md b/docs/publications/README.md new file mode 100644 index 0000000000000000000000000000000000000000..70d5e332f4fe140deeef62c7d0c99bcd5d994860 --- /dev/null +++ b/docs/publications/README.md @@ -0,0 +1,40 @@ +# Paper and Tech Report + +The reviewed PDFs are included in this release. Both files preserve the final +manuscript bytes; publication does not imply new experiments or peer acceptance. + +Author: **Kwangseob Ahn** +Affiliation: **INLEVEL9 / SEJONG UNIV.** +Contact: **oswarld@inlevel9.com** + +| Publication | Version | Download | +| --- | --- | --- | +| Vons: A Compact, Host-Controlled Decision Component for Agent Workflows | Paper v1.0, final editorial revision | [PDF · 15 pages](vons-paper-v1.0.pdf) | +| Vons: Compact Decision Models for Frontier-Agent Workflows | Technical Report v1.0.1 | [PDF · 20 pages](vons-tech-report-v1.0.1.pdf) | + +## Verification and provenance + +The final documents were reviewed against the frozen evidence packet and primary +references, including tables, figures, negative results and missing measurements. +Every PDF page was visually inspected. Independently checked local SHA-256 +values match the reviewed final versions. All pages render as US Letter, fonts +are embedded, and no PDF JavaScript, launch action or embedded attachment was +found. These are genuine Typst-generated PDFs; a clean source rebuild has not +been independently completed and is not claimed. + +| File | SHA-256 | +| --- | --- | +| `vons-paper-v1.0.pdf` | `ae0e09c60e17a98b52bb25c611b2489d10b18bd98d87974f2e8dd25d746a5fc8` | +| `vons-tech-report-v1.0.1.pdf` | `33076d9e52d4a073060066215dd390e3b0bf0cdef48c5ffb2a7c5189854254ed` | + +Only these exact PDFs are permitted by the public allowlist. Drafts, prompts, +claim ledgers, review conversations, source bundles and restricted benchmark data +remain excluded. The software has its own version, **0.1.0 research preview**. + +The [CC BY 4.0 article license](LICENSE.md) covers both manuscripts and their +original figures. Software rights are governed separately by the +[Vons Community and Commercial License 1.0](../../LICENSE); article licensing +does not grant commercial software rights. + +arXiv submission is in preparation. No arXiv identifier, DOI or acceptance is +announced. See the [arXiv preparation checklist](ARXIV.md). diff --git a/docs/publications/vons-paper-v1.0.pdf b/docs/publications/vons-paper-v1.0.pdf new file mode 100644 index 0000000000000000000000000000000000000000..352c4e366a9768e3fbbf6506bdb8ead9379088ab Binary files /dev/null and b/docs/publications/vons-paper-v1.0.pdf differ diff --git a/docs/publications/vons-tech-report-v1.0.1.pdf b/docs/publications/vons-tech-report-v1.0.1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4aa76596010ec35e0395839c376bcd3f55c93cd0 Binary files /dev/null and b/docs/publications/vons-tech-report-v1.0.1.pdf differ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..09e631708e4ef717062d18f8be3c9ac2c6f6d5ba --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools>=77.0.3"] +build-backend = "setuptools.build_meta" + +[project] +name = "vons" +version = "0.1.0" +description = "Small local decision models for frontier-agent workflows" +readme = "README.md" +requires-python = ">=3.10" +license = "LicenseRef-Vons-Community-Commercial-1.0" +license-files = ["LICENSE"] +authors = [{ name = "Vons contributors" }] +dependencies = [] + +[project.optional-dependencies] +train = ["numpy>=1.26", "torch>=2.2", "transformers>=4.40", "safetensors>=0.4"] +export = ["onnx>=1.16", "onnxruntime>=1.18", "onnxscript>=0.5"] +ollama = ["ollama>=0.3"] +dev = ["pytest>=8", "ruff>=0.6"] + +[project.scripts] +vons = "vons.cli:main" + +[tool.setuptools.packages.find] +include = ["vons*"] + +[tool.ruff] +line-length = 100 +target-version = "py310" +# Preserve acquired upstream evidence byte-for-byte; it is not Vons source. +extend-exclude = ["data/restricted"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/sdk/typescript/LICENSE b/sdk/typescript/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..02fdf205459235265f92d5fe07a7f143ca6c1ded --- /dev/null +++ b/sdk/typescript/LICENSE @@ -0,0 +1,104 @@ +Vons Community and Commercial License 1.0 +SPDX-License-Identifier: LicenseRef-Vons-Community-Commercial-1.0 + +1. Scope and definitions + +"Licensor" means each copyright holder who makes material available under this +license. "Software" means the Vons-owned source code, configuration, tests and +supporting software documentation accompanied by this license, including copies +and modifications. "You" means the individual or legal entity exercising the +permissions below, including persons acting on that entity's behalf. + +This license does not cover material with a separate license notice. In +particular, the paper and Tech Report and their original research figures are +licensed separately under CC BY 4.0. Third-party dependencies, upstream model +assets, datasets, trademarks and logos are not relicensed by this document. +No model weights or benchmark datasets are supplied with this source release. + +"Noncommercial Use" means personal learning, independent noncommercial +research, or noncommercial education, research or public-benefit activity by a +nonprofit or educational institution, where the activity is not undertaken for +commercial advantage, monetary compensation, or on behalf of a business. +Nonprofit or educational status alone does not make commercial activity +Noncommercial Use. Unrestricted donations that do not buy a service or benefit +do not by themselves make an otherwise qualifying activity commercial. + +"Commercial Use" includes paid consulting or client work; use in or to support +a monetized product, service, SaaS, API, hosted or managed service; internal +business operations; and evaluation, research or deployment by, for, or on +behalf of a for-profit enterprise. Contractors and affiliates acting for such +an enterprise require the same permission. Personal use by an employee outside +their employment, without business benefit, may qualify as Noncommercial Use. + +2. Free community permission + +Subject to this license, each Licensor grants You a worldwide, nonexclusive, +royalty-free copyright license to use, reproduce, study, modify, display and +distribute the Software solely for Noncommercial Use. You may provide access +to an otherwise qualifying noncommercial service under these conditions. +Only rights that a Licensor is entitled to grant are included. + +3. Redistribution and attribution + +When distributing copies or substantial portions of the Software, including +modified copies or a compiled form, You must: + +(a) retain this license, copyright notices and attribution notices; +(b) clearly identify Your modifications and preserve their provenance; +(c) identify the Vons version or source revision used where available; and +(d) convey these same use restrictions for the covered Software, without + representing that You can grant Commercial Use rights. + +Do not remove notices, misrepresent origin, or imply endorsement by a Licensor, +INLEVEL9, Sejong University or any other named institution. For research that +uses Vons, include an appropriate citation to the software and relevant paper +or Tech Report when reporting results. An unavailable DOI is not required. +Attribution or citation does not authorize an otherwise restricted use. + +4. Enterprise and commercial permission + +Commercial Use is not granted by this license. It requires a separate written +commercial agreement from the relevant Licensor before the activity begins. +There is no automatic enterprise evaluation or free-trial exception. Contact +oswarld@inlevel9.com to arrange the scope, fees and permitted entities. +This license sets no price, billing unit, service level or support commitment. +A commercial agreement applies only to the parties and scope it identifies. + +5. Outputs and independent rights + +This license claims no ownership or royalty over Your inference outputs merely +because they were produced using the Software. Outputs do not authorize an +unlicensed deployment and may contain material subject to independent rights. +Your inputs remain subject to their own rights and permissions. + +No trademark, logo, certification or endorsement rights are granted, except +the descriptive use needed to comply with the notices above. No patent license +is granted by this document. Statutory exceptions and other independently +obtained permissions are unaffected. This license does not revoke rights +previously granted under another license for an earlier copy, and does not +restrict separately licensed third-party components. + +6. Termination and cure + +Your permissions terminate automatically if You materially breach this +license. They are reinstated when the breach is cured within 30 days after +You first discover it, provided this is Your first breach. Otherwise written +reinstatement from the relevant Licensor is required. Termination does not +affect compliant recipients' direct permissions or obligations that by their +nature survive, including the notices and disclaimers below. + +7. Warranty disclaimer and liability + +TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE SOFTWARE IS PROVIDED +"AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND +NONINFRINGEMENT. THE LICENSORS ARE NOT LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY ARISING FROM THE SOFTWARE OR ITS USE, WHETHER IN CONTRACT, TORT OR +OTHERWISE. NOTHING HERE EXCLUDES LIABILITY THAT CANNOT LAWFULLY BE EXCLUDED. + +8. General + +Rights not expressly granted are reserved. No later license version applies +automatically. If a provision is unenforceable, the remaining provisions remain +in effect to the extent permitted by law. This is a source-available license +with use restrictions; it is not an OSI-approved open-source license. diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b9ba343943bd557d300653bf9abdf3f8e0ed0a54 --- /dev/null +++ b/sdk/typescript/README.md @@ -0,0 +1,49 @@ +# `@vons/decision-sdk` + +The TypeScript package mirrors the Vons Python request/response contract and +keeps KASI policy and execution in the host. It contains no tool executor and +does not treat a model-produced risk or confidence value as permission. + +The contract-only entry point remains small, while the optional `./onnx` +subpath uses pinned `onnxruntime-web@1.30.0` and +`@huggingface/tokenizers@0.2.0` for local browser inference. Applications +provide a model backend through `DecisionBackend`; the runtime helper verifies +bundle bytes, uses the declared WASM or WebGPU provider, and does not silently +fall back from an unavailable accelerator. +`callFingerprint` returns a canonical JSON consent key. Hosts that persist +consent for the Python adapter should SHA-256 this key before storage so the +cross-language fingerprint format remains identical. + +Node 22+ can load the source directly with its TypeScript type stripping during +early development. A release build should type-check and bundle the package +with the consuming application's toolchain. + +The local smoke harness covers both pilot bundles and both providers. Build it +and serve the repository root so the browser can fetch the ignored ORT runtime +files and the existing pilot artifacts: + +```sh +npm ci +npm run build:browser +cd ../.. +python3 -m http.server 8765 --bind 127.0.0.1 +``` + +Open `http://127.0.0.1:8765/sdk/typescript/demo/`. The smoke page reports +session-load and one-request timings, optional Chromium JS-heap observations, +provider evidence, and the contract response. These are diagnostic smoke +values, not the repeated benchmark protocol. + +## Source preview + +This package is private in npm metadata until an npm release is requested. +Its `files` allowlist includes source, this guide and the software license. Models, tokenizers, +datasets, generated demo JavaScript and dependencies are intentionally excluded +from the public source export. Build the demos locally and supply your own +reviewed model bundles before opening the smoke, benchmark or parity pages. + +The [Vons Community and Commercial License 1.0](LICENSE) permits qualifying +noncommercial use without a fee. Enterprise and other commercial use require a +separate written agreement; contact oswarld@inlevel9.com. This is source-available +software. Dependencies retain their own terms, and the research articles' CC BY +4.0 license does not grant commercial SDK rights. diff --git a/sdk/typescript/demo/benchmark.html b/sdk/typescript/demo/benchmark.html new file mode 100644 index 0000000000000000000000000000000000000000..8035dedeac86c74a55d1d101c942462500331ae0 --- /dev/null +++ b/sdk/typescript/demo/benchmark.html @@ -0,0 +1,17 @@ + + + + + + Vons ONNX Runtime Web benchmark + + +
+

Vons ONNX Runtime Web benchmark

+

Run with query parameters such as ?backend=direct&provider=wasm&sessions=5&warmup=10&repeats=100.

+ +
Waiting for benchmark parameters.
+
+ + + diff --git a/sdk/typescript/demo/benchmark.ts b/sdk/typescript/demo/benchmark.ts new file mode 100644 index 0000000000000000000000000000000000000000..ef9c9c3b449cc54bfd5705412335717c69dd9f2f --- /dev/null +++ b/sdk/typescript/demo/benchmark.ts @@ -0,0 +1,285 @@ +import { createOnnxWebBackend, type OnnxTimingSample } from "../src/onnx.ts"; + +type BackendName = "direct" | "diffusion"; +type ProviderName = "wasm" | "webgpu"; + +interface PerformanceWithMemory extends Performance { + memory?: { usedJSHeapSize: number }; +} + +interface NavigatorWithDeviceMemory extends Navigator { + deviceMemory?: number; +} + +interface BenchmarkSample extends Omit { + run_id: string; + session_start: number; + repeat: number; + case_id: string; + backend: BackendName; + requested_provider: ProviderName; + observed_provider: ProviderName | null; + observed_provider_status: string; + live_candidates: number | null; + allocated_candidates: number | null; + live_tokens: number | null; + sequence_length: number | null; + tokenizeMs: number | null; + prepareFeedMs: number | null; + inferenceAndReadbackMs: number | null; + postprocessMs: number | null; + totalRequestMs: number | null; + bundle_hash: string | null; + tokenizer_hash: string | null; + input_hash: string; + noise_hash: string | null; + js_heap_used_bytes_before: number | null; + js_heap_used_bytes_after: number | null; + status: "ok" | "error"; + error_kind?: string; +} + +interface SessionRecord { + session_start: number; + load_ms: number; + status: "ok" | "error"; + error_kind?: string; +} + +const output = document.querySelector("#output"); +if (!output) throw new Error("benchmark output is missing"); +const downloadButton = document.querySelector("#download-report"); +if (!downloadButton) throw new Error("benchmark download button is missing"); + +function showReport(report: unknown, fileName: string): void { + const text = JSON.stringify(report, null, 2); + output.textContent = text; + downloadButton.hidden = false; + downloadButton.onclick = () => { + const blob = new Blob([text], { type: "application/json" }); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = fileName; + anchor.click(); + URL.revokeObjectURL(url); + }; + console.log(text); +} + +const manifestUrls = { + direct: new URL("../../../artifacts/pilot/bundle-manifest-v1.json", import.meta.url), + diffusion: new URL("../../../artifacts/pilot-diffusion/bundle-manifest-v1.json", import.meta.url), +}; + +const manifestHashes = { + direct: "bd39eea2b8d5ed5f4d57daefdd4822c4e2c4817a68c4a42bef850b5edcbc97d", + diffusion: "df0e7cae640c6604fb517a88d08dbdaa2454d6b3dbbf5708926349cf490b9607", +}; + +const heapBytes = (): number | null => (performance as PerformanceWithMemory).memory?.usedJSHeapSize ?? null; + +function parameter(name: string, fallback: number): number { + const raw = new URLSearchParams(location.search).get(name); + if (raw === null) return fallback; + const value = Number(raw); + if (!Number.isInteger(value) || value < 1) throw new RangeError(`${name} must be a positive integer`); + return value; +} + +function textParameter(name: string, fallback: string): string { + return new URLSearchParams(location.search).get(name) ?? fallback; +} + +function stableJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + const entries = Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b)); + return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(",")}}`; +} + +async function sha256(value: string | ArrayBufferLike): Promise { + const bytes = typeof value === "string" ? new TextEncoder().encode(value) : new Uint8Array(value); + const digest = await crypto.subtle.digest("SHA-256", bytes as unknown as BufferSource); + return Array.from(new Uint8Array(digest), (item) => item.toString(16).padStart(2, "0")).join(""); +} + +// Frozen from NumPy default_rng(7).standard_normal((1, 32), dtype=float32). +// The array, rather than a language-specific seed algorithm, is the parity input. +const FROZEN_NOISE_SEED_7 = Float32Array.from([ + 1.52196932, -1.14410579, 1.15016162, -1.78465319, -0.77149874, 1.01916146, + -0.926134586, -0.415745318, -0.0629121512, 0.513251185, -0.285995334, + 1.98480856, 1.30306911, -0.00464721443, 1.17548203, 1.34566987, + -0.171809882, -2.4133966, -0.0993812904, -0.979565322, 0.942675471, + 0.452982962, 0.53610903, 0.494613141, 1.94684041, 0.207542002, + -1.64390635, -0.542155981, -0.384397745, -0.410856962, 0.946411192, + 0.979099393, +]); + +function percentile(values: number[], fraction: number): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const index = (sorted.length - 1) * fraction; + const lower = Math.floor(index); + const upper = Math.ceil(index); + if (lower === upper) return sorted[lower]; + return sorted[lower] + (sorted[upper] - sorted[lower]) * (index - lower); +} + +async function run(): Promise { + const backendName = textParameter("backend", "direct") as BackendName; + const provider = textParameter("provider", "wasm") as ProviderName; + if (!(["direct", "diffusion"] as string[]).includes(backendName)) throw new RangeError("backend must be direct or diffusion"); + if (!(["wasm", "webgpu"] as string[]).includes(provider)) throw new RangeError("provider must be wasm or webgpu"); + const sessions = parameter("sessions", 1); + const warmup = parameter("warmup", 1); + const repeats = parameter("repeats", 3); + const caseCount = parameter("cases", 20); + const runId = crypto.randomUUID(); + const cases = Array.from({ length: caseCount }, (_, index) => ({ + caseId: `case-${index + 1}`, + state: `browser benchmark state ${index + 1}`, + })); + const request = (caseIndex: number) => ({ + state: cases[caseIndex % cases.length].state, + questions: [{ id: cases[caseIndex % cases.length].caseId, type: "choice" as const, prompt: "Choose the next action", options: ["continue", "stop", "inspect", "defer"] }], + seed: 7, + }); + const inputHashes = new Map(cases.map((item, index) => [item.caseId, stableJson(request(index))])); + const hashedInputs = new Map(); + for (const [caseId, value] of inputHashes) hashedInputs.set(caseId, await sha256(value)); + const noise = backendName === "diffusion" ? FROZEN_NOISE_SEED_7 : null; + const noiseHash = noise ? await sha256(noise.buffer) : null; + const browserEnvironment = { + user_agent: navigator.userAgent, + platform: navigator.platform || null, + hardware_concurrency: navigator.hardwareConcurrency ?? null, + device_memory_gb: (navigator as NavigatorWithDeviceMemory).deviceMemory ?? null, + cross_origin_isolated: globalThis.crossOriginIsolated, + }; + const samples: BenchmarkSample[] = []; + const sessionRecords: SessionRecord[] = []; + let runtimeInfo: Awaited>["runtimeInfo"] | null = null; + + for (let sessionStart = 1; sessionStart <= sessions; sessionStart += 1) { + let timing: OnnxTimingSample | undefined; + const started = performance.now(); + let backend: Awaited> | undefined; + try { + backend = await createOnnxWebBackend({ + manifestUrl: manifestUrls[backendName], + expectedManifestSha256: manifestHashes[backendName], + provider, + wasmPaths: "/sdk/typescript/node_modules/onnxruntime-web/dist/", + wasmNumThreads: 1, + onTiming: (sample) => { timing = sample; }, + initialNoise: backendName === "diffusion" ? () => new Float32Array(noise as Float32Array) : undefined, + }); + runtimeInfo = backend.runtimeInfo; + sessionRecords.push({ session_start: sessionStart, load_ms: performance.now() - started, status: "ok" }); + for (let repeat = 0; repeat < warmup; repeat += 1) await backend.decide(request(repeat)); + for (let repeat = 0; repeat < repeats; repeat += 1) { + const caseValue = cases[repeat % cases.length]; + const before = heapBytes(); + timing = undefined; + try { + await backend.decide(request(repeat)); + if (!timing) throw new Error("timing hook did not produce a sample"); + samples.push({ + ...timing, + run_id: runId, + session_start: sessionStart, + repeat, + case_id: caseValue.caseId, + backend: backendName, + requested_provider: provider, + observed_provider: null, + observed_provider_status: "session_created; execution_partition_not_exposed", + live_candidates: timing.live_candidates, + allocated_candidates: timing.allocated_candidates, + live_tokens: timing.live_tokens, + sequence_length: timing.sequence_length, + bundle_hash: backend.manifestHash, + tokenizer_hash: backend.manifest.files.find((item) => item.path.endsWith("tokenizer.json"))?.sha256 ?? null, + input_hash: hashedInputs.get(caseValue.caseId) as string, + noise_hash: noiseHash, + js_heap_used_bytes_before: before, + js_heap_used_bytes_after: heapBytes(), + status: "ok", + }); + } catch (error) { + samples.push({ + questionId: caseValue.caseId, + tokenizeMs: null, + prepareFeedMs: null, + inferenceAndReadbackMs: null, + postprocessMs: null, + totalRequestMs: null, + run_id: runId, + session_start: sessionStart, + repeat, + case_id: caseValue.caseId, + backend: backendName, + requested_provider: provider, + observed_provider: null, + observed_provider_status: "request_failed; execution_partition_not_exposed", + live_candidates: null, + allocated_candidates: null, + live_tokens: null, + sequence_length: null, + bundle_hash: backend.manifestHash, + tokenizer_hash: backend.manifest.files.find((item) => item.path.endsWith("tokenizer.json"))?.sha256 ?? null, + input_hash: hashedInputs.get(caseValue.caseId) as string, + noise_hash: noiseHash, + js_heap_used_bytes_before: before, + js_heap_used_bytes_after: heapBytes(), + status: "error", + error_kind: error instanceof Error ? `${error.name}: ${error.message}` : String(error), + }); + } + } + } catch (error) { + sessionRecords.push({ session_start: sessionStart, load_ms: performance.now() - started, status: "error", error_kind: error instanceof Error ? `${error.name}: ${error.message}` : String(error) }); + } finally { + await backend?.dispose(); + } + } + + const successful = samples.filter((sample) => sample.status === "ok"); + const memoryMaximum = (values: Array): number | null => { + const available = values.filter((value): value is number => value !== null); + return available.length === 0 ? null : Math.max(...available); + }; + const report = { + schema: "vons.browser-benchmark/v1", + run_id: runId, + backend: backendName, + requested_provider: provider, + sessions, + warmup_excluded_per_session: warmup, + repeats_per_session: repeats, + cases: caseCount, + environment: browserEnvironment, + runtime_info: runtimeInfo, + session_start_semantics: "new ORT session on one page; module and browser caches may persist", + execution_provider_evidence: "requested provider and session creation only; graph partition status is not exposed by this harness", + session_records: sessionRecords, + samples, + summary: { + successful_samples: successful.length, + failed_samples: samples.length - successful.length, + total_request_ms_p50: percentile(successful.flatMap((sample) => sample.totalRequestMs === null ? [] : [sample.totalRequestMs]), 0.5), + total_request_ms_p95: percentile(successful.flatMap((sample) => sample.totalRequestMs === null ? [] : [sample.totalRequestMs]), 0.95), + inference_and_readback_ms_p50: percentile(successful.flatMap((sample) => sample.inferenceAndReadbackMs === null ? [] : [sample.inferenceAndReadbackMs]), 0.5), + inference_and_readback_ms_p95: percentile(successful.flatMap((sample) => sample.inferenceAndReadbackMs === null ? [] : [sample.inferenceAndReadbackMs]), 0.95), + js_heap_before_max: memoryMaximum(successful.map((sample) => sample.js_heap_used_bytes_before)), + js_heap_after_max: memoryMaximum(successful.map((sample) => sample.js_heap_used_bytes_after)), + memory_note: "performance.memory before/after observations only; not a peak or GPU-memory measurement", + }, + }; + showReport(report, `vons-${backendName}-${provider}-${runId}.json`); +} + +void run().catch((error) => { + showReport({ schema: "vons.browser-benchmark/v1", status: "error", error: error instanceof Error ? `${error.name}: ${error.message}` : String(error) }, `vons-browser-benchmark-error-${crypto.randomUUID()}.json`); +}); diff --git a/sdk/typescript/demo/index.html b/sdk/typescript/demo/index.html new file mode 100644 index 0000000000000000000000000000000000000000..2fac63c345e99c213d61b1dd1ed2cf597932b548 --- /dev/null +++ b/sdk/typescript/demo/index.html @@ -0,0 +1,20 @@ + + + + + + Vons ONNX Runtime Web smoke + + +
+

Vons ONNX Runtime Web smoke

+

This harness is intentionally local-only. It reports runtime evidence; it is not a benchmark.

+ + + + +
Not run.
+
+ + + diff --git a/sdk/typescript/demo/main.ts b/sdk/typescript/demo/main.ts new file mode 100644 index 0000000000000000000000000000000000000000..b982f5c93deecf42252f0de450d9551f300b5669 --- /dev/null +++ b/sdk/typescript/demo/main.ts @@ -0,0 +1,78 @@ +import { createOnnxWebBackend } from "../src/onnx.ts"; + +const outputElement = document.querySelector("#output"); +if (!outputElement) throw new Error("smoke output element is missing"); +const output = outputElement; + +interface PerformanceWithMemory extends Performance { + memory?: { usedJSHeapSize: number }; +} + +const heapBytes = (): number | null => (performance as PerformanceWithMemory).memory?.usedJSHeapSize ?? null; + +const manifestUrls = { + direct: new URL("../../../artifacts/pilot/bundle-manifest-v1.json", import.meta.url), + diffusion: new URL("../../../artifacts/pilot-diffusion/bundle-manifest-v1.json", import.meta.url), +}; + +const manifestHashes = { + direct: "bd39eea2b8d5ed5f4d57daefdd4822c4e2c4817a68c4a42bef850b5edcbc97d", + diffusion: "df0e7cae640c6604fb517a88d08dbdaa2454d6b3dbbf5708926349cf490b9607", +}; + +function show(value: unknown): void { + output.textContent = typeof value === "string" ? value : JSON.stringify(value, null, 2); +} + +async function run(provider: "wasm" | "webgpu", backendName: "direct" | "diffusion"): Promise { + const started = performance.now(); + let backend: Awaited> | undefined; + try { + backend = await createOnnxWebBackend({ + manifestUrl: manifestUrls[backendName], + expectedManifestSha256: manifestHashes[backendName], + provider, + wasmPaths: "/sdk/typescript/node_modules/onnxruntime-web/dist/", + wasmNumThreads: 1, + }); + const initialized = performance.now(); + const beforeMemory = heapBytes(); + const response = await backend.decide({ + state: "browser smoke state", + questions: [{ id: "smoke", type: "choice", prompt: "Choose the next action", options: ["continue", "stop"] }], + seed: 7, + }); + const completed = performance.now(); + show({ + pass: true, + backend: backendName, + runtime: backend.runtimeInfo, + load_ms: initialized - started, + inference_ms: completed - initialized, + total_ms: completed - started, + js_heap_used_bytes_before: beforeMemory, + js_heap_used_bytes_after: heapBytes(), + response, + }); + } catch (error) { + show({ + pass: false, + backend: backendName, + provider, + elapsed_ms: performance.now() - started, + error: error instanceof Error ? `${error.name}: ${error.message}` : String(error), + }); + } finally { + await backend?.dispose(); + } +} + +for (const button of Array.from(document.querySelectorAll("button[data-backend]"))) { + button.addEventListener("click", () => { + const value = button.dataset.backend; + if (value === "direct") void run("wasm", "direct"); + else if (value === "diffusion") void run("wasm", "diffusion"); + else if (value === "direct-webgpu") void run("webgpu", "direct"); + else if (value === "diffusion-webgpu") void run("webgpu", "diffusion"); + }); +} diff --git a/sdk/typescript/demo/parity.html b/sdk/typescript/demo/parity.html new file mode 100644 index 0000000000000000000000000000000000000000..e31f891cdf97d7d0e0aa814b1846af5337db027d --- /dev/null +++ b/sdk/typescript/demo/parity.html @@ -0,0 +1,20 @@ + + + + + + Vons ONNX Runtime Web parity matrix + + +
+

Vons ONNX Runtime Web parity matrix

+

This local harness runs the frozen 23-fixture contract matrix. It is not a quality benchmark.

+ + + + +
Not run.
+
+ + + diff --git a/sdk/typescript/demo/parity.ts b/sdk/typescript/demo/parity.ts new file mode 100644 index 0000000000000000000000000000000000000000..b12da8d020cff37bed58c463f6f8f79eaf0fa8c9 --- /dev/null +++ b/sdk/typescript/demo/parity.ts @@ -0,0 +1,239 @@ +import { createOnnxWebBackend, type OnnxTimingSample, type OnnxTraceSample } from "../src/onnx.ts"; +import type { DecisionRequest, DecisionResponse, QuestionType } from "../src/index.ts"; + +export type ParityBackend = "direct" | "diffusion"; +export type ParityProvider = "wasm" | "webgpu"; + +export interface ParityFixture { + id: string; + request: DecisionRequest; + expectedFailure?: string; + matrix: { candidate_count: number; family: string; target_tokens: string }; +} + +export interface ParityRecord { + fixture_id: string; + input_hash: string; + request: DecisionRequest; + matrix: ParityFixture["matrix"]; + status: "ok" | "expected_error" | "unexpected_error"; + response?: DecisionResponse; + timings?: OnnxTimingSample[]; + traces?: OnnxTraceSample[]; + error?: string; +} + +const manifestUrls: Record = { + direct: new URL("../../../artifacts/pilot/bundle-manifest-v1.json", import.meta.url), + diffusion: new URL("../../../artifacts/pilot-diffusion/bundle-manifest-v1.json", import.meta.url), +}; + +const manifestHashes: Record = { + direct: "bd39eea2b8d5ed5f4d57daefdd4822c4e2c4817a68c4a42bef850b5edcbc97d", + diffusion: "df0e7cae640c6604fb517a88d08dbdaa2454d6b3dbbf5708926349cf490b9607", +}; + +const FROZEN_NOISE_SEED_7 = Float32Array.from([ + 1.52196932, -1.14410579, 1.15016162, -1.78465319, -0.77149874, 1.01916146, + -0.926134586, -0.415745318, -0.0629121512, 0.513251185, -0.285995334, 1.98480856, + 1.30306911, -0.00464721443, 1.17548203, 1.34566987, -0.171809882, -2.4133966, + -0.0993812904, -0.979565322, 0.942675471, 0.452982962, 0.53610903, 0.494613141, + 1.94684041, 0.207542002, -1.64390635, -0.542155981, -0.384397745, -0.410856962, + 0.946411192, 0.979099393, +]); + +function options(count: number, label: string): string[] { + return Array.from({ length: count }, (_, index) => `${label} candidate ${index + 1}`); +} + +function question( + id: string, + type: QuestionType, + prompt: string, + values: string[], +): DecisionRequest["questions"][number] { + return type === "score" + ? { id, type, prompt, options: values, rubric: ["poor", "fair", "good", "very good", "excellent"] } + : { id, type, prompt, options: values }; +} + +function fixture( + id: string, + family: string, + targetTokens: string, + candidateCount: number, + state: DecisionRequest["state"], + item: DecisionRequest["questions"][number], + expectedFailure?: string, +): ParityFixture { + return { + id, + request: { state, questions: [item], seed: 7 }, + expectedFailure, + matrix: { candidate_count: candidateCount, family, target_tokens: targetTokens }, + }; +} + +export function buildParityFixtures(): ParityFixture[] { + const result: ParityFixture[] = []; + const counts = [2, 4, 8, 16, 32]; + for (const count of counts) { + result.push(fixture(`short-${count}`, "choice", "short", count, "ready", question(`short-${count}-q`, "choice", "Choose the next action", options(count, "short")))); + result.push(fixture(`unicode-${count}`, "choice", "unicode", count, { locale: "ko-KR", state: `검토 중인 화면 🌐 / пункт ${count}` }, question(`unicode-${count}-q`, "choice", "다음에 선택할 항목은 무엇입니까?", options(count, "unicode/다음")))); + result.push(fixture(`score-${count}`, "score", "unsupported", count, { workflow: "quality", count, labels: ["α", "β", "γ"] }, question(`score-${count}-q`, "score", "Rate the candidate outcome", options(count, "score")), "score_head_unsupported")); + const repeats = Math.max(2, Math.floor(60 / count)); + result.push(fixture(`long-${count}`, "choice", "near-aggregate-limit", count, "state ".repeat(50), question(`long-${count}-q`, "choice", "Select the most relevant detailed action", options(count, "detail ".repeat(repeats))))); + } + result.push(fixture("boolean-2", "boolean", "boolean", 2, "permission check", question("boolean-q", "boolean", "Should the host ask for confirmation?", []))); + result.push({ + id: "multi-question", + request: { + state: { page: "settings", text: "Review before any external action" }, + questions: [ + question("multi-choice", "choice", "Which next step is safest?", options(4, "multi")), + question("multi-boolean", "boolean", "Is confirmation required?", []), + ], + seed: 7, + }, + matrix: { candidate_count: 4, family: "multi-question", target_tokens: "mixed" }, + }); + result.push(fixture( + "overflow-explicit", + "overflow", + "over-limit", + 2, + "x ".repeat(700), + question("overflow-q", "choice", "This request must fail explicitly", options(2, "overflow")), + "request_validation_or_tokenization_overflow", + )); + return result; +} + +function stableJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + const entries = Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b)); + return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(",")}}`; +} + +async function sha256(value: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)); + return Array.from(new Uint8Array(digest), (item) => item.toString(16).padStart(2, "0")).join(""); +} + +export async function runParityCell(backendName: ParityBackend, provider: ParityProvider): Promise> { + const fixtures = buildParityFixtures(); + const records: ParityRecord[] = []; + let timings: OnnxTimingSample[] = []; + let traces: OnnxTraceSample[] = []; + const backend = await createOnnxWebBackend({ + manifestUrl: manifestUrls[backendName], + expectedManifestSha256: manifestHashes[backendName], + provider, + wasmPaths: "/sdk/typescript/node_modules/onnxruntime-web/dist/", + wasmNumThreads: 1, + initialNoise: backendName === "diffusion" ? () => new Float32Array(FROZEN_NOISE_SEED_7) : undefined, + onTiming: (sample) => { timings.push(sample); }, + onTrace: (sample) => { traces.push(sample); }, + }); + try { + for (const item of fixtures) { + const inputHash = await sha256(stableJson(item.request)); + timings = []; + traces = []; + try { + const response = await backend.decide(item.request); + records.push({ fixture_id: item.id, input_hash: inputHash, request: item.request, matrix: item.matrix, status: "ok", response, timings, traces }); + } catch (error) { + const message = error instanceof Error ? `${error.name}: ${error.message}` : String(error); + records.push({ + fixture_id: item.id, + input_hash: inputHash, + request: item.request, + matrix: item.matrix, + status: item.expectedFailure ? "expected_error" : "unexpected_error", + timings, + traces, + error: message, + }); + } + } + } finally { + await backend.dispose(); + } + return { + schema: "vons.browser-parity/v1", + backend: backendName, + requested_provider: provider, + fixture_count: fixtures.length, + runtime_info: backend.runtimeInfo, + manifest_hash: backend.manifestHash, + tokenizer_hash: backend.manifest.files.find((item) => item.path.endsWith("tokenizer.json"))?.sha256 ?? null, + noise_seed: backendName === "diffusion" ? 7 : null, + noise_values: backendName === "diffusion" ? Array.from(FROZEN_NOISE_SEED_7) : null, + records, + limitations: [ + "This harness checks contract-valid responses and reproducible fixture inputs; it does not establish task accuracy.", + "A browser-vs-Python numeric parity claim requires separately saved CPU reference outputs for the same input and noise hashes.", + "WebGPU provider selection does not expose graph partition evidence in this runtime harness.", + ], + }; +} + +if (typeof document !== "undefined") { + const runButton = document.querySelector("#run"); + const output = document.querySelector("#output"); + const backendSelect = document.querySelector("#backend"); + const providerSelect = document.querySelector("#provider"); + const downloadButton = document.querySelector("#download"); + if (!runButton || !output || !backendSelect || !providerSelect || !downloadButton) throw new Error("parity harness controls are missing"); + let lastReport: Record | null = null; + const displayReport = (report: Record): void => { + const records = Array.isArray(report.records) ? report.records as Array> : []; + const statusCounts = records.reduce>((counts, record) => { + const status = String(record.status ?? "unknown"); + counts[status] = (counts[status] ?? 0) + 1; + return counts; + }, {}); + output.textContent = JSON.stringify({ + schema: report.schema, + backend: report.backend, + requested_provider: report.requested_provider, + fixture_count: report.fixture_count, + manifest_hash: report.manifest_hash, + tokenizer_hash: report.tokenizer_hash, + noise_seed: report.noise_seed, + status_counts: statusCounts, + errors: records.filter((record) => record.status !== "ok").map((record) => ({ fixture_id: record.fixture_id, status: record.status, error: record.error })), + note: "Full parity JSON, including request hashes, timings, traces, and noise values, is available through Download report.", + }, null, 2); + }; + downloadButton.addEventListener("click", () => { + if (!lastReport) return; + const backend = backendSelect.value as ParityBackend; + const provider = providerSelect.value as ParityProvider; + const blob = new Blob([JSON.stringify(lastReport, null, 2)], { type: "application/json" }); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = `vons-parity-${backend}-${provider}.json`; + link.click(); + URL.revokeObjectURL(url); + }); + runButton.addEventListener("click", async () => { + runButton.disabled = true; + downloadButton.disabled = true; + lastReport = null; + output.textContent = "Running..."; + try { + const report = await runParityCell(backendSelect.value as ParityBackend, providerSelect.value as ParityProvider); + lastReport = report; + downloadButton.disabled = false; + displayReport(report); + } catch (error) { + output.textContent = JSON.stringify({ schema: "vons.browser-parity/v1", status: "error", error: String(error) }, null, 2); + } finally { + runButton.disabled = false; + } + }); +} diff --git a/sdk/typescript/package-lock.json b/sdk/typescript/package-lock.json new file mode 100644 index 0000000000000000000000000000000000000000..08f3cd6f355b1be213947f4346406661c55b4542 --- /dev/null +++ b/sdk/typescript/package-lock.json @@ -0,0 +1,652 @@ +{ + "name": "@vons/decision-sdk", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@vons/decision-sdk", + "version": "0.1.0", + "dependencies": { + "@huggingface/tokenizers": "0.2.0", + "onnxruntime-web": "1.30.0" + }, + "devDependencies": { + "esbuild": "0.28.2" + }, + "engines": { + "node": ">=22" + }, + "license": "SEE LICENSE IN LICENSE" + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tokenizers": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.2.0.tgz", + "integrity": "sha512-LidMHe1FpcSYH4vcSjXooda34pC0M8m1gmDOv7SQi6Iv+ib1zX7J3sHBCC9/3FPCkn2k1b55FneVqMRxrr99pg==", + "license": "Apache-2.0" + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@types/node": { + "version": "26.6.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.2.tgz", + "integrity": "sha512-X1P21scMv4zGKLYqjdGjaKa7COa0RKVYYZZN/NfvLQ1JegxFhdhpZG/Lyn8AXx6CDUavKAd11v6BvfpkDByK8g==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/onnxruntime-common": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.30.0.tgz", + "integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==", + "license": "MIT" + }, + "node_modules/onnxruntime-web": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.30.0.tgz", + "integrity": "sha512-q0y+JrrtukXSzsBWEMccVfqX25LRmosXHF+CaRJmg8pZClzcV7svNc4rKY3jL02Vb7QmRMDs1SigqR4CXAfKYQ==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.30.0", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, + "node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "license": "MIT" + } + } +} diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json new file mode 100644 index 0000000000000000000000000000000000000000..a248485a1e608973a6cb1e1727649e421e602fbd --- /dev/null +++ b/sdk/typescript/package.json @@ -0,0 +1,33 @@ +{ + "name": "@vons/decision-sdk", + "version": "0.1.0", + "description": "Contract and host-boundary SDK for Vons decision models", + "type": "module", + "dependencies": { + "@huggingface/tokenizers": "0.2.0", + "onnxruntime-web": "1.30.0" + }, + "exports": { + ".": "./src/index.ts", + "./onnx": "./src/onnx.ts" + }, + "scripts": { + "test": "node --experimental-strip-types --test test/index.test.mjs test/onnx.test.mjs test/parity.test.mjs", + "build:browser": "esbuild demo/main.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/main.js", + "build:benchmark": "esbuild demo/benchmark.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/benchmark.js", + "build:parity": "esbuild demo/parity.ts --bundle --format=esm --platform=browser --target=es2022 --outfile=demo/parity.js" + }, + "engines": { + "node": ">=22" + }, + "devDependencies": { + "esbuild": "0.28.2" + }, + "private": true, + "files": [ + "src/", + "README.md", + "LICENSE" + ], + "license": "SEE LICENSE IN LICENSE" +} diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..047c5a67d2119b5c6cdf696a32cc7356c8d5332e --- /dev/null +++ b/sdk/typescript/src/index.ts @@ -0,0 +1,255 @@ +export type QuestionType = "choice" | "boolean" | "score"; +export type Backend = "direct" | "diffusion"; +export type ResponseStatus = "ok" | "abstain"; + +const validQuestionTypes = new Set(["choice", "boolean", "score"]); + +export interface Question { + id: string; + type: QuestionType; + prompt: string; + options: string[]; + rubric?: string[]; +} + +export interface DecisionRequest { + state: string | Record; + questions: Question[]; + backend?: Backend; + seed?: number; +} + +export interface OptionProbability { + option: string; + probability: number; +} + +export interface QuestionAnswer { + question_id: string; + choice: string | null; + probabilities: OptionProbability[]; + confidence: number; + status: ResponseStatus; + abstain_reason?: string | null; +} + +export interface DecisionResponse { + answers: QuestionAnswer[]; + backend: Backend; + model_id: string; + metadata?: Record; +} + +export type ToolRisk = "low" | "medium" | "high" | "critical"; + +export interface ToolCall { + name: string; + arguments: Record; +} + +export interface KASIProposal { + calls?: ToolCall[]; + confidence: number; + risk?: string | null; + response?: string | null; +} + +export type KASIAction = "call" | "clarify" | "confirm" | "refuse" | "respond"; + +export interface KASIActionDecision { + action: KASIAction; + calls: ToolCall[]; + prompt: string | null; + reason: string; +} + +export interface DecisionBackend { + readonly backend: Backend; + decide(request: DecisionRequest): Promise; +} + +const toolNamePattern = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/; +const validRisks = new Set(["low", "medium", "high", "critical"]); +const riskRank: Record = { low: 0, medium: 1, high: 2, critical: 3 }; + +function assertFiniteUnit(value: number, name: string): void { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new TypeError(`${name} must be finite and in [0, 1]`); + } +} + +function assertToolName(name: string): void { + if (typeof name !== "string" || !toolNamePattern.test(name)) { + throw new TypeError("tool name must match [A-Za-z0-9][A-Za-z0-9_.:-]{0,127}"); + } +} + +function normalizeRisk(value: string | null | undefined): ToolRisk | null { + return value && validRisks.has(value.trim().toLowerCase() as ToolRisk) + ? (value.trim().toLowerCase() as ToolRisk) + : null; +} + +function stableJson(value: unknown): string { + if (value === null || typeof value !== "object") { + const primitive = JSON.stringify(value); + if (primitive === undefined) throw new TypeError("arguments must be JSON-compatible"); + return primitive; + } + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + const entries = Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b)); + return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(",")}}`; +} + +export function callFingerprint(call: ToolCall): string { + assertToolName(call.name); + // SHA-256 is supplied by the host in production. The portable SDK key is a + // canonical consent key; callers may hash it before persistence. + return stableJson({ name: call.name, arguments: call.arguments }); +} + +export function validateRequest(request: DecisionRequest, maxQuestions = 8, maxTokens = 512): void { + if (!request || typeof request !== "object") throw new TypeError("request must be an object"); + if (!Number.isInteger(maxQuestions) || maxQuestions < 1) throw new TypeError("maxQuestions must be a positive integer"); + if (!Number.isInteger(maxTokens) || maxTokens < 1) throw new TypeError("maxTokens must be a positive integer"); + if (typeof request.state !== "string" && (!request.state || typeof request.state !== "object" || Array.isArray(request.state))) { + throw new TypeError("state must be a string or object"); + } + if (typeof request.state === "string" && request.state.trim().length === 0) { + throw new TypeError("state must not be empty"); + } + if (!Array.isArray(request.questions) || request.questions.length < 1 || request.questions.length > maxQuestions) { + throw new TypeError(`questions must contain 1..${maxQuestions} items`); + } + const ids = new Set(); + for (const question of request.questions) { + if (!question || typeof question !== "object") throw new TypeError("questions must contain objects"); + if (typeof question.id !== "string" || !question.id || ids.has(question.id)) throw new TypeError("question ids must be unique and non-empty"); + ids.add(question.id); + if (!validQuestionTypes.has(question.type)) throw new TypeError(`question ${question.id} has an invalid type`); + if (typeof question.prompt !== "string" || !question.prompt.trim()) throw new TypeError(`question ${question.id} has an empty prompt`); + if (!Array.isArray(question.options) || question.options.some((option) => typeof option !== "string" || !option.trim())) { + throw new TypeError(`question ${question.id} options must be strings`); + } + if (question.type === "boolean") { + if (question.options.length > 0 && JSON.stringify(question.options) !== JSON.stringify(["true", "false"])) { + throw new TypeError("boolean questions must use true/false options"); + } + } else if (!Array.isArray(question.options) || question.options.length < 2 || question.options.length > 32) { + throw new TypeError(`question ${question.id} must contain 2..32 options`); + } + if (new Set(question.options).size !== question.options.length) throw new TypeError("question options must be unique"); + if (question.rubric !== undefined && !Array.isArray(question.rubric)) { + throw new TypeError("question rubric must be a list"); + } + if (question.type === "score" && (!question.rubric || question.rubric.length < 2 || question.rubric.length > 10)) { + throw new TypeError("score questions must contain a 2..10 item rubric"); + } + if (question.type === "score" && question.rubric?.some((item) => typeof item !== "string")) { + throw new TypeError("score question rubric items must be strings"); + } + if (question.type !== "score" && question.rubric?.length) { + throw new TypeError(`only score question ${question.id} may provide rubric`); + } + } + // Tokenizer-specific budget checks belong to the runtime adapter. A character + // estimate here would reject valid WordPiece inputs and cannot account for + // the aggregate state/question/all-candidate contract. Keep maxTokens in the + // signature for API compatibility and let adapters enforce their exact limit. + void maxTokens; +} + +export function validateResponse(response: DecisionResponse): void { + if (!response || typeof response !== "object") throw new TypeError("response must be an object"); + if (typeof response.model_id !== "string" || !response.model_id.trim()) throw new TypeError("model_id is required"); + if (response.backend !== "direct" && response.backend !== "diffusion") throw new TypeError("invalid backend"); + if (!Array.isArray(response.answers)) throw new TypeError("answers must be a list"); + if (response.metadata !== undefined && (!response.metadata || typeof response.metadata !== "object" || Array.isArray(response.metadata))) { + throw new TypeError("metadata must be an object"); + } + const ids = new Set(); + for (const answer of response.answers) { + if (!answer || typeof answer !== "object") throw new TypeError("answers must contain objects"); + if (typeof answer.question_id !== "string" || !answer.question_id || ids.has(answer.question_id)) throw new TypeError("answer question ids must be unique and non-empty"); + ids.add(answer.question_id); + if (!Array.isArray(answer.probabilities)) throw new TypeError("probabilities must be a list"); + assertFiniteUnit(answer.confidence, "confidence"); + const options = new Set(); + let total = 0; + for (const item of answer.probabilities) { + if (!item || typeof item !== "object" || typeof item.option !== "string" || !item.option) { + throw new TypeError("probability options must be non-empty strings"); + } + if (options.has(item.option)) throw new TypeError("probability options must be unique"); + options.add(item.option); + assertFiniteUnit(item.probability, "probability"); + total += item.probability; + } + if (answer.probabilities.length > 0 && Math.abs(total - 1) > 1e-6) throw new TypeError("probabilities must sum to 1"); + if (answer.status !== "ok" && answer.status !== "abstain") throw new TypeError("invalid response status"); + if (answer.choice !== null && typeof answer.choice !== "string") throw new TypeError("choice must be a string or null"); + if (answer.abstain_reason !== undefined && answer.abstain_reason !== null && typeof answer.abstain_reason !== "string") { + throw new TypeError("abstain_reason must be a string or null"); + } + if (answer.status === "ok" && !answer.choice) throw new TypeError("ok answers require choice"); + if (answer.status === "abstain" && (answer.choice !== null || !answer.abstain_reason?.trim())) { + throw new TypeError("abstain answers require a reason and no choice"); + } + if (answer.status === "ok" && answer.probabilities.length > 0 && !options.has(answer.choice!)) { + throw new TypeError("choice must be present in probabilities"); + } + } +} + +export class KASIAdapter { + private readonly policy: ReadonlyMap; + private readonly threshold: number; + + constructor(policy: Record, threshold = 0.55) { + if (!Number.isFinite(threshold) || threshold <= 0 || threshold > 1) throw new TypeError("invalid threshold"); + this.threshold = threshold; + const entries: [string, ToolRisk][] = []; + for (const [name, risk] of Object.entries(policy)) { + assertToolName(name); + if (!validRisks.has(risk)) throw new TypeError(`invalid risk for ${name}`); + entries.push([name, risk]); + } + this.policy = new Map(entries); + } + + decide(proposal: KASIProposal, confirmedCalls: ReadonlySet = new Set()): KASIActionDecision { + assertFiniteUnit(proposal.confidence, "confidence"); + const calls = proposal.calls ?? []; + if (calls.length > 0 && proposal.response) throw new TypeError("proposal cannot contain both calls and response"); + if (proposal.response && !(proposal.calls?.length)) { + if (proposal.confidence >= this.threshold) return { action: "respond", calls: [], prompt: proposal.response, reason: "direct_response" }; + } + if (proposal.confidence < this.threshold) return { action: "clarify", calls, prompt: "Please provide the missing information before acting.", reason: "low_confidence" }; + if (calls.length === 0) return { action: "refuse", calls, prompt: "No safe candidate action is available.", reason: "no_proposed_call" }; + const advisory = normalizeRisk(proposal.risk); + if (proposal.risk !== null && proposal.risk !== undefined && !advisory) { + return { action: "confirm", calls, prompt: "Confirm action with unknown model risk.", reason: "unknown_model_risk" }; + } + for (const call of calls) { + assertToolName(call.name); + const hostRisk = this.policy.get(call.name); + if (!hostRisk) return { action: "refuse", calls: [], prompt: "Tool is not registered by the host.", reason: "unregistered_tool" }; + const needsConsent = !advisory || Math.max(riskRank[hostRisk], riskRank[advisory]) >= riskRank.high; + if (needsConsent && !confirmedCalls.has(callFingerprint(call))) { + return { action: "confirm", calls, prompt: `Confirm action: ${call.name}(${stableJson(call.arguments)}).`, reason: "unconfirmed_or_unknown_risk" }; + } + } + return { action: "call", calls, prompt: null, reason: "approved_by_host_policy" }; + } +} + +export function selectRuntime(preferred: "wasm" | "webgpu" = "wasm"): "wasm" | "webgpu" { + if (preferred === "webgpu") { + const scope = globalThis as { navigator?: { gpu?: unknown }; gpu?: unknown }; + const gpu = scope.navigator?.gpu ?? scope.gpu; + if (!gpu) throw new Error("WebGPU is unavailable; choose wasm explicitly"); + return "webgpu"; + } + if (typeof WebAssembly === "undefined") throw new Error("WebAssembly is unavailable"); + return "wasm"; +} diff --git a/sdk/typescript/src/onnx.ts b/sdk/typescript/src/onnx.ts new file mode 100644 index 0000000000000000000000000000000000000000..c0e927b3cc9c1f884252f3ca600bf15afaf92cae --- /dev/null +++ b/sdk/typescript/src/onnx.ts @@ -0,0 +1,606 @@ +import { Tokenizer } from "@huggingface/tokenizers"; + +import { + type Backend, + type DecisionBackend, + type DecisionRequest, + type DecisionResponse, + type Question, + type QuestionAnswer, + validateRequest, + validateResponse, +} from "./index.ts"; + +export type OnnxExecutionProvider = "wasm" | "webgpu"; + +export interface BundleFileRecord { + path: string; + role: string; + bytes: number; + sha256: string; + runtime_loaded: boolean; + model_asset: boolean; +} + +export interface BundleManifest { + schema_version: "vons.bundle.manifest/v1"; + manifest: { path: string; sha256: string | null; self_hash_excluded: boolean }; + metadata: { + backend: Backend; + model_id?: string; + option_count?: number; + sequence_length?: number; + input_names?: string[]; + output_names?: string[]; + graph?: { + external_data_locations?: string[]; + inputs?: Array<{ name: string; dtype: string; shape: Array | null }>; + outputs?: Array<{ name: string; dtype: string; shape: Array | null }>; + }; + [key: string]: unknown; + }; + files: BundleFileRecord[]; + summary: { + model_asset_bytes: number; + complete_download_bytes: number; + [key: string]: unknown; + }; + [key: string]: unknown; +} + +export interface TokenizedCandidate { + inputIds: number[]; + attentionMask: number[]; + tokenTypeIds: number[]; +} + +export interface OnnxWebBackendOptions { + /** Use a URL when the manifest is served beside the browser bundle. */ + manifestUrl?: string | URL; + /** Supply a previously fetched manifest when the caller controls loading. */ + manifest?: BundleManifest; + /** Pin the exact manifest bytes expected by a release or measurement run. */ + expectedManifestSha256?: string; + /** Required with an in-memory manifest unless the browser has a useful base URI. */ + baseUrl?: string | URL; + provider?: OnnxExecutionProvider; + wasmPaths?: string | Record; + wasmNumThreads?: number; + abstainThreshold?: number; + answerabilityThreshold?: number; + /** Override Direct's live candidate count for padding-cost experiments. */ + directOptionSlots?: number; + /** Freeze the exact noise array when a cross-runtime comparison needs it. */ + initialNoise?: (length: number, seed: number | undefined, questionIndex: number) => Float32Array; + /** Optional instrumentation hook; it is not part of the decision response contract. */ + onTiming?: (sample: OnnxTimingSample) => void; + /** Optional local parity trace; disabled by default and not part of the response contract. */ + onTrace?: (sample: OnnxTraceSample) => void; + fetch?: typeof fetch; + signal?: AbortSignal; +} + +export interface OnnxRuntimeInfo { + provider: OnnxExecutionProvider; + ortVersion: string | null; + wasmNumThreads: number | null; + webgpuAdapterRequested: boolean; + webgpuDeviceRequested: boolean; +} + +export interface OnnxTimingSample { + questionId: string; + live_candidates: number; + allocated_candidates: number; + live_tokens: number; + sequence_length: number; + tokenizeMs: number; + prepareFeedMs: number; + inferenceAndReadbackMs: number; + postprocessMs: number; + totalRequestMs: number; +} + +export interface OnnxTraceSample { + questionId: string; + inputIds: number[]; + attentionMask: number[]; + tokenTypeIds: number[]; + optionMask: number[]; + initialNoise: number[] | null; + rawScores: number[]; + answerabilityLogit: number; + probabilities: number[]; + answer: QuestionAnswer; +} + +interface OrtTensor { + data: ArrayLike; + dims: readonly number[]; +} + +interface OrtSession { + run(feeds: Record): Promise>; + release(): Promise; + inputNames: readonly string[]; + outputNames: readonly string[]; +} + +interface OrtModule { + Tensor: new (type: string, data: ArrayLike | ArrayLike, dims: readonly number[]) => unknown; + InferenceSession: { create(model: ArrayBufferLike, options?: Record): Promise }; + env: { + wasm: { wasmPaths?: string | Record; numThreads?: number }; + versions?: { common?: string; web?: string }; + }; +} + +interface GpuLike { + requestAdapter(): Promise<{ requestDevice(): Promise } | null>; +} + +interface LoadedAssets { + manifest: BundleManifest; + manifestHash: string; + model: Uint8Array; + tokenizer: Tokenizer; + externalData: Array<{ path: string; data: Uint8Array }>; +} + +const EXPECTED_SCHEMA = "vons.bundle.manifest/v1"; +const REQUIRED_DIRECT_INPUTS = ["input_ids", "attention_mask", "token_type_ids", "option_mask"]; +const REQUIRED_DIFFUSION_INPUTS = [...REQUIRED_DIRECT_INPUTS, "initial_noise"]; + +function assertFiniteUnit(value: number, name: string): void { + if (!Number.isFinite(value) || value < 0 || value > 1) throw new TypeError(`${name} must be finite and in [0, 1]`); +} + +function stableJson(value: unknown): string { + if (value === null || typeof value !== "object") { + const primitive = JSON.stringify(value); + if (primitive === undefined) throw new TypeError("state must be JSON-compatible"); + return primitive; + } + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + const entries = Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b)); + return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(",")}}`; +} + +export function serializeState(state: DecisionRequest["state"]): string { + return typeof state === "string" ? state : stableJson(state); +} + +export function candidateText(state: DecisionRequest["state"], question: Question, option: string): string { + return `${serializeState(state)}\nQuestion: ${question.prompt}\nCandidate: ${option}`; +} + +export function candidateListText(state: DecisionRequest["state"], question: Question, options: readonly string[]): string { + return `${serializeState(state)}\nQuestion: ${question.prompt}\nCandidates:\n${options.join("\n")}`; +} + +function assertManifest(manifest: BundleManifest): void { + if (!manifest || manifest.schema_version !== EXPECTED_SCHEMA) { + throw new TypeError(`unsupported bundle manifest schema: ${manifest?.schema_version ?? "missing"}`); + } + if (!manifest.metadata || (manifest.metadata.backend !== "direct" && manifest.metadata.backend !== "diffusion")) { + throw new TypeError("bundle manifest metadata.backend must be direct or diffusion"); + } + if (!Array.isArray(manifest.files) || manifest.files.length === 0) throw new TypeError("bundle manifest has no files"); + const paths = new Set(); + for (const item of manifest.files) { + if (!item || typeof item.path !== "string" || item.path.length === 0 || paths.has(item.path)) { + throw new TypeError(`bundle manifest contains an invalid or duplicate path: ${item?.path ?? "missing"}`); + } + if (item.path.startsWith("/") || item.path.includes("..") || /^[A-Za-z]:[\\/]/.test(item.path)) { + throw new TypeError(`bundle manifest path must be bundle-relative: ${item.path}`); + } + if (!Number.isSafeInteger(item.bytes) || item.bytes < 0 || !/^[0-9a-f]{64}$/.test(item.sha256)) { + throw new TypeError(`bundle manifest has invalid bytes or sha256: ${item.path}`); + } + paths.add(item.path); + } +} + +function getBaseUrl(options: OnnxWebBackendOptions): URL { + if (options.baseUrl) return new URL(options.baseUrl.toString(), globalThis.location?.href ?? "http://localhost/"); + if (options.manifestUrl) return new URL(options.manifestUrl.toString(), globalThis.location?.href ?? "http://localhost/"); + if (typeof globalThis.location?.href === "string") return new URL(globalThis.location.href); + throw new TypeError("baseUrl or manifestUrl is required when running outside a browser"); +} + +function resolveAssetUrl(base: URL, path: string): URL { + if ( + path.startsWith("/") || + path.includes("\\") || + path.includes("..") || + /%(?:2e|2f|5c)/i.test(path) || + /[?#]/.test(path) || + /^[A-Za-z]:[\\/]/.test(path) || + /^[A-Za-z][A-Za-z+.-]*:/.test(path) + ) { + throw new TypeError(`bundle asset path must be relative: ${path}`); + } + const resolved = new URL(path, base); + const basePath = base.pathname.endsWith("/") ? base.pathname : `${base.pathname}/`; + if (resolved.origin !== base.origin || !resolved.pathname.startsWith(basePath)) { + throw new TypeError(`bundle asset path escapes its manifest directory: ${path}`); + } + return resolved; +} + +async function sha256(bytes: Uint8Array): Promise { + if (!globalThis.crypto?.subtle) throw new Error("Web Crypto SHA-256 is unavailable"); + const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes as unknown as BufferSource); + return Array.from(new Uint8Array(digest), (value) => value.toString(16).padStart(2, "0")).join(""); +} + +async function fetchBytes(fetchImpl: typeof fetch, url: URL, signal?: AbortSignal): Promise { + const response = await fetchImpl(url, { signal }); + if (!response.ok) throw new Error(`failed to fetch ${url}: HTTP ${response.status}`); + return new Uint8Array(await response.arrayBuffer()); +} + +async function loadManifest(options: OnnxWebBackendOptions, fetchImpl: typeof fetch): Promise<{ manifest: BundleManifest; manifestHash: string; base: URL }> { + if (options.expectedManifestSha256 !== undefined && !/^[0-9a-f]{64}$/.test(options.expectedManifestSha256)) { + throw new TypeError("expectedManifestSha256 must be a lowercase SHA-256 digest"); + } + if (options.manifest) { + assertManifest(options.manifest); + const manifestBytes = new TextEncoder().encode(stableJson(options.manifest)); + const manifestHash = await sha256(manifestBytes); + if (options.expectedManifestSha256 !== undefined && manifestHash !== options.expectedManifestSha256) { + throw new Error(`bundle manifest SHA-256 mismatch: ${manifestHash}`); + } + return { manifest: options.manifest, base: getBaseUrl(options), manifestHash }; + } + if (!options.manifestUrl) throw new TypeError("manifest or manifestUrl is required"); + const url = new URL(options.manifestUrl.toString(), globalThis.location?.href ?? "http://localhost/"); + const bytes = await fetchBytes(fetchImpl, url, options.signal); + let manifest: BundleManifest; + try { + manifest = JSON.parse(new TextDecoder().decode(bytes)) as BundleManifest; + } catch (error) { + throw new TypeError(`bundle manifest is not valid JSON: ${String(error)}`); + } + assertManifest(manifest); + const manifestHash = await sha256(bytes); + if (options.expectedManifestSha256 !== undefined && manifestHash !== options.expectedManifestSha256) { + throw new Error(`bundle manifest SHA-256 mismatch: ${manifestHash}`); + } + return { manifest, base: new URL(".", url), manifestHash }; +} + +async function loadAssets(options: OnnxWebBackendOptions): Promise { + const fetchImpl = options.fetch ?? globalThis.fetch?.bind(globalThis); + if (!fetchImpl) throw new Error("fetch is unavailable; provide options.fetch"); + const { manifest, manifestHash, base } = await loadManifest(options, fetchImpl); + const cache = new Map(); + const getFile = async (record: BundleFileRecord): Promise => { + const cached = cache.get(record.path); + if (cached) return cached; + const bytes = await fetchBytes(fetchImpl, resolveAssetUrl(base, record.path), options.signal); + if (bytes.byteLength !== record.bytes) throw new Error(`bundle byte count mismatch: ${record.path}`); + const actual = await sha256(bytes); + if (actual !== record.sha256) throw new Error(`bundle SHA-256 mismatch: ${record.path}`); + cache.set(record.path, bytes); + return bytes; + }; + const graph = manifest.files.find((item) => item.role === "model_graph"); + if (!graph) throw new Error("bundle manifest has no model graph"); + const tokenizerJson = manifest.files.find((item) => item.role === "tokenizer" && item.path.endsWith("tokenizer.json")); + const tokenizerConfig = manifest.files.find((item) => item.role === "tokenizer" && item.path.endsWith("tokenizer_config.json")); + if (!tokenizerJson || !tokenizerConfig) throw new Error("bundle manifest must include tokenizer.json and tokenizer_config.json"); + const model = await getFile(graph); + const tokenizer = new Tokenizer( + JSON.parse(new TextDecoder().decode(await getFile(tokenizerJson))) as object, + JSON.parse(new TextDecoder().decode(await getFile(tokenizerConfig))) as object, + ); + const locations = manifest.metadata.graph?.external_data_locations ?? []; + const externalData: Array<{ path: string; data: Uint8Array }> = []; + for (const location of locations) { + const record = manifest.files.find((item) => item.role === "model_external_data" && item.path === location); + if (!record) throw new Error(`external-data location is missing from the manifest: ${location}`); + externalData.push({ path: location, data: await getFile(record) }); + } + return { manifest, manifestHash, model, tokenizer, externalData }; +} + +function encodeCandidate(tokenizer: Tokenizer, text: string, sequenceLength: number): TokenizedCandidate { + const encoded = tokenizer.encode(text, { return_token_type_ids: true }); + const inputIds = [...encoded.ids]; + const attentionMask = [...encoded.attention_mask]; + const tokenTypeIds = [...encoded.token_type_ids]; + if (inputIds.length !== attentionMask.length || inputIds.length !== tokenTypeIds.length) { + throw new Error("tokenizer returned arrays with different lengths"); + } + if (inputIds.length > sequenceLength) { + throw new RangeError(`tokenized candidate has ${inputIds.length} tokens; the bundle limit is ${sequenceLength}`); + } + if (inputIds.some((value) => !Number.isSafeInteger(value) || value < 0)) throw new Error("tokenizer returned an invalid token id"); + return { inputIds, attentionMask, tokenTypeIds }; +} + +export function tokenizeCandidates( + tokenizer: Tokenizer, + state: DecisionRequest["state"], + question: Question, + options: readonly string[], + sequenceLength: number, +): TokenizedCandidate[] { + return options.map((option) => encodeCandidate(tokenizer, candidateText(state, question, option), sequenceLength)); +} + +function padCandidates(candidates: readonly TokenizedCandidate[], slots: number, sequenceLength: number, paddingId: number): { + inputIds: BigInt64Array; + attentionMask: BigInt64Array; + tokenTypeIds: BigInt64Array; + optionMask: Uint8Array; +} { + if (!Number.isInteger(slots) || slots < candidates.length || slots < 1) throw new RangeError("invalid candidate slot count"); + const inputIds = new BigInt64Array(slots * sequenceLength); + const attentionMask = new BigInt64Array(slots * sequenceLength); + const tokenTypeIds = new BigInt64Array(slots * sequenceLength); + const optionMask = new Uint8Array(slots); + inputIds.fill(BigInt(paddingId)); + for (let optionIndex = 0; optionIndex < candidates.length; optionIndex += 1) { + const candidate = candidates[optionIndex]; + optionMask[optionIndex] = 1; + const offset = optionIndex * sequenceLength; + for (let tokenIndex = 0; tokenIndex < candidate.inputIds.length; tokenIndex += 1) { + inputIds[offset + tokenIndex] = BigInt(candidate.inputIds[tokenIndex]); + attentionMask[offset + tokenIndex] = BigInt(candidate.attentionMask[tokenIndex]); + tokenTypeIds[offset + tokenIndex] = BigInt(candidate.tokenTypeIds[tokenIndex]); + } + } + return { inputIds, attentionMask, tokenTypeIds, optionMask }; +} + +export function seededNoise(length: number, seed: number | undefined, questionIndex: number): Float32Array { + if (!Number.isInteger(length) || length < 1) throw new RangeError("noise length must be positive"); + let state = ((seed ?? 0) ^ (questionIndex + 1) * 0x9e3779b9) >>> 0; + const result = new Float32Array(length); + const nextUnit = (): number => { + state = (Math.imul(state ^ (state >>> 16), 0x45d9f3b) + 0x27100001) >>> 0; + // Open interval (0, 1) avoids log(0) while keeping the stream deterministic. + return (state + 1) / 4294967297; + }; + for (let index = 0; index < length; index += 1) { + const radius = Math.sqrt(-2 * Math.log(nextUnit())); + const angle = 2 * Math.PI * nextUnit(); + result[index] = radius * Math.cos(angle); + if (index + 1 < length) { + result[index + 1] = radius * Math.sin(angle); + index += 1; + } + } + return result; +} + +function sigmoid(value: number): number { + if (!Number.isFinite(value)) throw new Error("model returned a non-finite answerability logit"); + return value >= 0 ? 1 / (1 + Math.exp(-value)) : Math.exp(value) / (1 + Math.exp(value)); +} + +function softmax(values: readonly number[]): number[] { + if (values.length === 0 || values.some((value) => Number.isNaN(value))) throw new Error("model returned invalid option scores"); + const maximum = Math.max(...values); + if (!Number.isFinite(maximum)) throw new Error("model returned no finite option score"); + const exponentials = values.map((value) => Math.exp(value - maximum)); + const total = exponentials.reduce((sum, value) => sum + value, 0); + if (!Number.isFinite(total) || total <= 0) throw new Error("could not normalize model option scores"); + return exponentials.map((value) => value / total); +} + +function tensor(runtime: OrtModule, type: string, data: ArrayLike | ArrayLike, dims: readonly number[]): unknown { + return new runtime.Tensor(type, data, dims); +} + +function tensorValues(output: OrtTensor | undefined, expectedLength: number, name: string): number[] { + if (!output || !output.dims) throw new Error(`missing output tensor: ${name}`); + const values = Array.from(output.data, Number); + if (values.length !== expectedLength) throw new Error(`output ${name} has ${values.length} values; expected ${expectedLength}`); + return values; +} + +async function loadOrt(provider: OnnxExecutionProvider, options: OnnxWebBackendOptions): Promise<{ runtime: OrtModule; info: OnnxRuntimeInfo }> { + if (provider === "webgpu") { + const gpu = (globalThis.navigator as Navigator & { gpu?: GpuLike } | undefined)?.gpu; + if (!gpu) throw new Error("WebGPU is unavailable; choose wasm explicitly"); + const adapter = await gpu.requestAdapter(); + if (!adapter) throw new Error("WebGPU adapter request returned null"); + await adapter.requestDevice(); + } + const runtime = (provider === "webgpu" + ? await import("onnxruntime-web/webgpu") + : await import("onnxruntime-web/wasm")) as unknown as OrtModule; + if (options.wasmPaths) runtime.env.wasm.wasmPaths = options.wasmPaths; + if (provider === "wasm") { + if (options.wasmNumThreads !== undefined) { + if (!Number.isInteger(options.wasmNumThreads) || options.wasmNumThreads < 1) throw new TypeError("wasmNumThreads must be a positive integer"); + runtime.env.wasm.numThreads = options.wasmNumThreads; + } + } + return { + runtime, + info: { + provider, + ortVersion: runtime.env.versions?.web ?? runtime.env.versions?.common ?? null, + wasmNumThreads: provider === "wasm" ? runtime.env.wasm.numThreads ?? null : null, + webgpuAdapterRequested: provider === "webgpu", + webgpuDeviceRequested: provider === "webgpu", + }, + }; +} + +export class OnnxWebBackend implements DecisionBackend { + readonly backend: Backend; + readonly runtimeInfo: OnnxRuntimeInfo; + readonly manifest: BundleManifest; + readonly manifestHash: string; + private readonly tokenizer: Tokenizer; + private readonly runtime: OrtModule; + private readonly session: OrtSession; + private readonly sequenceLength: number; + private readonly optionCount: number; + private readonly paddingId: number; + private readonly abstainThreshold: number; + private readonly answerabilityThreshold: number; + private readonly directOptionSlots: number | undefined; + private readonly initialNoise: OnnxWebBackendOptions["initialNoise"]; + private readonly onTiming: OnnxWebBackendOptions["onTiming"]; + private readonly onTrace: OnnxWebBackendOptions["onTrace"]; + private disposed = false; + private constructor(assets: LoadedAssets, runtime: OrtModule, runtimeInfo: OnnxRuntimeInfo, session: OrtSession, options: OnnxWebBackendOptions) { + this.backend = assets.manifest.metadata.backend; + this.runtimeInfo = runtimeInfo; + this.manifest = assets.manifest; + this.manifestHash = assets.manifestHash; + this.tokenizer = assets.tokenizer; + this.runtime = runtime; + this.session = session; + this.sequenceLength = assets.manifest.metadata.sequence_length ?? 512; + this.optionCount = assets.manifest.metadata.option_count ?? 32; + this.paddingId = assets.tokenizer.token_to_id("[PAD]") ?? 0; + this.abstainThreshold = options.abstainThreshold ?? 0.55; + this.answerabilityThreshold = options.answerabilityThreshold ?? 0.5; + this.directOptionSlots = options.directOptionSlots; + this.initialNoise = options.initialNoise; + this.onTiming = options.onTiming; + this.onTrace = options.onTrace; + } + + static async create(options: OnnxWebBackendOptions = {}): Promise { + const assets = await loadAssets(options); + const provider = options.provider ?? "wasm"; + assertFiniteUnit(options.abstainThreshold ?? 0.55, "abstainThreshold"); + assertFiniteUnit(options.answerabilityThreshold ?? 0.5, "answerabilityThreshold"); + const { runtime, info } = await loadOrt(provider, options); + const expectedInputs = assets.manifest.metadata.backend === "diffusion" ? REQUIRED_DIFFUSION_INPUTS : REQUIRED_DIRECT_INPUTS; + const modelInputs = assets.manifest.metadata.input_names ?? expectedInputs; + if (JSON.stringify(modelInputs) !== JSON.stringify(expectedInputs)) throw new Error("bundle input contract is not supported by this backend"); + const outputNames = assets.manifest.metadata.output_names ?? (assets.manifest.metadata.backend === "diffusion" ? ["scores", "answerability"] : ["logits", "answerability"]); + const session = await runtime.InferenceSession.create(assets.model.buffer, { + executionProviders: [provider], + externalData: assets.externalData, + }); + try { + if (session.inputNames.length !== expectedInputs.length || expectedInputs.some((name) => !session.inputNames.includes(name))) { + throw new Error(`loaded session inputs do not match the manifest: ${session.inputNames.join(",")}`); + } + if (outputNames.some((name) => !session.outputNames.includes(name))) throw new Error("loaded session outputs do not match the manifest"); + } catch (error) { + await session.release(); + throw error; + } + return new OnnxWebBackend(assets, runtime, info, session, options); + } + + async decide(request: DecisionRequest): Promise { + if (this.disposed) throw new Error("ONNX Runtime session has been disposed"); + validateRequest(request); + const answers: QuestionAnswer[] = []; + for (const [questionIndex, question] of request.questions.entries()) { + const requestStarted = performance.now(); + if (question.type === "score") { + throw new Error("score questions are not supported by this candidate-selection ONNX head"); + } + const options = question.type === "boolean" && question.options.length === 0 ? ["true", "false"] : question.options; + const slots = this.backend === "diffusion" ? this.optionCount : this.directOptionSlots ?? options.length; + if (options.length === 0 || options.length > this.optionCount || slots > this.optionCount) { + throw new RangeError(`question ${question.id} has unsupported candidate count`); + } + const aggregateTokenCount = this.tokenizer.encode(candidateListText(request.state, question, options)).ids.length; + if (aggregateTokenCount > this.sequenceLength) { + throw new RangeError(`question ${question.id} exceeds the ${this.sequenceLength}-token aggregate input budget (tokens=${aggregateTokenCount})`); + } + const tokenizeStarted = performance.now(); + const encoded = tokenizeCandidates(this.tokenizer, request.state, question, options, this.sequenceLength); + const tokenized = performance.now(); + const padded = padCandidates(encoded, slots, this.sequenceLength, this.paddingId); + let traceNoise: Float32Array | null = null; + const feeds: Record = { + input_ids: tensor(this.runtime, "int64", padded.inputIds, [1, slots, this.sequenceLength]), + attention_mask: tensor(this.runtime, "int64", padded.attentionMask, [1, slots, this.sequenceLength]), + token_type_ids: tensor(this.runtime, "int64", padded.tokenTypeIds, [1, slots, this.sequenceLength]), + option_mask: tensor(this.runtime, "bool", padded.optionMask, [1, slots]), + }; + if (this.backend === "diffusion") { + const noise = this.initialNoise?.(this.optionCount, request.seed, questionIndex) ?? seededNoise(this.optionCount, request.seed, questionIndex); + if (!(noise instanceof Float32Array) || noise.length !== this.optionCount) throw new TypeError("initialNoise must return a Float32Array with option_count values"); + traceNoise = noise; + feeds.initial_noise = tensor(this.runtime, "float32", noise, [1, this.optionCount]); + } + const inferenceStarted = performance.now(); + const outputs = await this.session.run(feeds); + const readbackCompleted = performance.now(); + const scoreName = this.backend === "diffusion" ? "scores" : "logits"; + const scores = tensorValues(outputs[scoreName], slots, scoreName).slice(0, options.length); + const probabilities = softmax(scores); + const answerabilityLogit = tensorValues(outputs.answerability, 1, "answerability")[0]; + const answerability = sigmoid(answerabilityLogit); + const choiceIndex = probabilities.reduce((best, value, index) => value > probabilities[best] ? index : best, 0); + const confidence = probabilities[choiceIndex] * answerability; + const abstainReason = answerability < this.answerabilityThreshold + ? "answerability_below_threshold" + : confidence < this.abstainThreshold ? "confidence_below_threshold" : null; + const answer: QuestionAnswer = { + question_id: question.id, + choice: abstainReason ? null : options[choiceIndex], + probabilities: options.map((option, index) => ({ option, probability: probabilities[index] })), + confidence, + status: abstainReason ? "abstain" : "ok", + abstain_reason: abstainReason, + }; + answers.push(answer); + this.onTrace?.({ + questionId: question.id, + inputIds: Array.from(padded.inputIds, Number), + attentionMask: Array.from(padded.attentionMask, Number), + tokenTypeIds: Array.from(padded.tokenTypeIds, Number), + optionMask: Array.from(padded.optionMask, Number), + initialNoise: traceNoise === null ? null : Array.from(traceNoise), + rawScores: scores, + answerabilityLogit, + probabilities, + answer, + }); + const postprocessCompleted = performance.now(); + this.onTiming?.({ + questionId: question.id, + live_candidates: encoded.length, + allocated_candidates: slots, + live_tokens: encoded.reduce((total, candidate) => total + candidate.inputIds.length, 0), + sequence_length: this.sequenceLength, + tokenizeMs: tokenized - tokenizeStarted, + prepareFeedMs: inferenceStarted - tokenized, + inferenceAndReadbackMs: readbackCompleted - inferenceStarted, + postprocessMs: postprocessCompleted - readbackCompleted, + totalRequestMs: postprocessCompleted - requestStarted, + }); + } + const response: DecisionResponse = { + answers, + backend: this.backend, + model_id: this.manifest.metadata.model_id ?? `vons-${this.backend}-onnx-web`, + metadata: { + provider: this.runtimeInfo.provider, + ort_version: this.runtimeInfo.ortVersion, + sequence_length: this.sequenceLength, + candidate_slots: this.backend === "diffusion" ? this.optionCount : this.directOptionSlots ?? "live", + tokenizer: "@huggingface/tokenizers@0.2.0", + }, + }; + validateResponse(response); + return response; + } + + async dispose(): Promise { + if (this.disposed) return; + this.disposed = true; + await this.session.release(); + } +} + +export async function createOnnxWebBackend(options: OnnxWebBackendOptions = {}): Promise { + return OnnxWebBackend.create(options); +} diff --git a/sdk/typescript/test/index.test.mjs b/sdk/typescript/test/index.test.mjs new file mode 100644 index 0000000000000000000000000000000000000000..358df8d934b0b85cbd356f376f7bbcc30e11d031 --- /dev/null +++ b/sdk/typescript/test/index.test.mjs @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +const sdk = await import("../src/index.ts"); + +test("host policy refuses unregistered tools and requires exact high-risk consent", () => { + const adapter = new sdk.KASIAdapter({ unlock: "high" }); + const first = { name: "unlock", arguments: { target: "lab" } }; + const second = { name: "unlock", arguments: { target: "vault" } }; + assert.equal(adapter.decide({ calls: [second], confidence: 0.9, risk: "low" }).action, "confirm"); + assert.equal(adapter.decide({ calls: [second], confidence: 0.9, risk: "low" }, new Set([sdk.callFingerprint(first)])).action, "confirm"); + assert.equal(adapter.decide({ calls: [second], confidence: 0.9, risk: "low" }, new Set([sdk.callFingerprint(second)])).action, "call"); + assert.equal(new sdk.KASIAdapter({}).decide({ calls: [{ name: "read", arguments: {} }], confidence: 0.9, risk: "low" }).action, "refuse"); +}); + +test("request and response validators enforce the shared contract", () => { + assert.doesNotThrow(() => sdk.validateRequest({ + state: "ready", + questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["a", "b"] }], + })); + assert.throws(() => sdk.validateResponse({ + answers: [{ question_id: "q", choice: "a", probabilities: [{ option: "a", probability: 0.5 }], confidence: 0.9, status: "ok" }], + backend: "direct", + model_id: "test", + })); + assert.throws(() => sdk.validateRequest({ + state: "ready", + questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["a", "b"], rubric: ["bad"] }], + }), /only score question/); + assert.doesNotThrow(() => sdk.validateRequest({ + state: "x".repeat(2000), + questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["a", "b"] }], + })); + assert.doesNotThrow(() => sdk.validateRequest({ + state: "x".repeat(200), + questions: Array.from({ length: 8 }, (_, index) => ({ + id: `q-${index}`, + type: "choice", + prompt: "Pick", + options: ["a", "b"], + })), + })); + assert.doesNotThrow(() => sdk.validateRequest({ + state: "ready", + questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["x".repeat(2000), "b"] }], + })); + assert.throws(() => sdk.validateRequest({ + state: "ready", + questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["", "b"] }], + }), /options must be strings/); + assert.doesNotThrow(() => sdk.validateRequest({ + state: "ready", + questions: [{ id: "q", type: "choice", prompt: "Pick", options: ["a", "b"] }], + }, 8, 512)); +}); + +test("runtime selection is explicit", () => { + assert.equal(sdk.selectRuntime("wasm"), "wasm"); + assert.throws(() => sdk.selectRuntime("webgpu"), /WebGPU is unavailable/); + const previousGpu = globalThis.gpu; + globalThis.gpu = {}; + assert.equal(sdk.selectRuntime("webgpu"), "webgpu"); + if (previousGpu === undefined) delete globalThis.gpu; + else globalThis.gpu = previousGpu; +}); diff --git a/sdk/typescript/test/onnx.test.mjs b/sdk/typescript/test/onnx.test.mjs new file mode 100644 index 0000000000000000000000000000000000000000..b6872788eb34447ceb3292fce6d478af46d362b4 --- /dev/null +++ b/sdk/typescript/test/onnx.test.mjs @@ -0,0 +1,101 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +const onnx = await import("../src/onnx.ts"); + +// Deliberately synthetic: public contract tests must not require pilot weights. +function testManifest() { + const record = (path, role, bytes = 4) => ({ + path, role, bytes, sha256: "1".repeat(64), runtime_loaded: true, model_asset: true, + }); + return { + schema_version: "vons.bundle.manifest/v1", + manifest: { path: "bundle-manifest.json", sha256: null, self_hash_excluded: true }, + metadata: { backend: "direct", option_count: 2, sequence_length: 32 }, + files: [ + record("model.onnx", "model_graph"), + record("tokenizer.json", "tokenizer"), + record("tokenizer_config.json", "tokenizer"), + ], + summary: { model_asset_bytes: 12, complete_download_bytes: 12 }, + }; +} + +test("structured state and the training text template are deterministic", () => { + const question = { id: "q", type: "choice", prompt: "Pick", options: ["A"] }; + assert.equal(onnx.serializeState({ z: 1, a: ["x", 2] }), '{"a":["x",2],"z":1}'); + assert.equal(onnx.candidateText({ z: 1, a: ["x", 2] }, question, "A"), '{"a":["x",2],"z":1}\nQuestion: Pick\nCandidate: A'); +}); + +test("default diffusion noise is deterministic Gaussian, not a bounded uniform surrogate", () => { + const first = onnx.seededNoise(32, 7, 0); + const second = onnx.seededNoise(32, 7, 0); + assert.deepEqual(Array.from(first), Array.from(second)); + assert.ok(first.some((value) => value > 1 || value < -1)); + assert.ok(Array.from(first).every(Number.isFinite)); +}); + +test("tokenizer ids, masks, and overflow behavior are explicit", async () => { + const { Tokenizer } = await import("@huggingface/tokenizers"); + const tokenizer = new Tokenizer( + { + version: "1.0", + added_tokens: [], + normalizer: { type: "BertNormalizer", clean_text: true, handle_chinese_chars: true, strip_accents: null, lowercase: true }, + pre_tokenizer: { type: "BertPreTokenizer" }, + post_processor: { type: "BertProcessing", sep: ["[SEP]", 2], cls: ["[CLS]", 1] }, + decoder: { type: "WordPiece", prefix: "##", cleanup: true }, + model: { type: "WordPiece", unk_token: "[UNK]", continuing_subword_prefix: "##", max_input_chars_per_word: 100, + vocab: { "[UNK]": 0, "[CLS]": 1, "[SEP]": 2, ready: 3, question: 4, pick: 5, candidate: 6, ":": 7, a: 8, b: 9, "/": 10, "?": 11, "中": 12, "文": 13 } }, + }, + { tokenizer_class: "BertTokenizer", unk_token: "[UNK]", cls_token: "[CLS]", sep_token: "[SEP]", model_max_length: 512 }, + ); + const question = { id: "q", type: "choice", prompt: "Pick", options: ["A/B?", "中文"] }; + const values = onnx.tokenizeCandidates(tokenizer, "ready", question, question.options, 512); + assert.deepEqual(values[0].inputIds, [1, 3, 4, 7, 5, 6, 7, 8, 10, 9, 11, 2]); + assert.deepEqual(values[1].inputIds, [1, 3, 4, 7, 5, 6, 7, 12, 13, 2]); + assert.equal(values[0].inputIds.length, values[0].attentionMask.length); + assert.deepEqual(new Set(values[0].attentionMask), new Set([1])); + assert.throws(() => onnx.tokenizeCandidates(tokenizer, "x", question, ["x".repeat(10000)], 4), /bundle limit/); +}); + +test("manifest integrity is checked before a runtime session is created", async () => { + const manifest = testManifest(); + const original = manifest.files.find((item) => item.role === "model_graph"); + original.sha256 = "0".repeat(64); + const fetch = async () => new Response(new Uint8Array(original.bytes), { status: 200 }); + await assert.rejects( + onnx.createOnnxWebBackend({ manifest, baseUrl: "https://example.test/bundle/", fetch }), + /SHA-256 mismatch/, + ); +}); + +test("release runs can pin manifest bytes and encoded traversal is rejected", async () => { + const manifest = testManifest(); + await assert.rejects( + onnx.createOnnxWebBackend({ + manifest, + baseUrl: "https://example.test/bundle/", + expectedManifestSha256: "0".repeat(64), + fetch: async () => new Response(new Uint8Array(), { status: 200 }), + }), + /manifest SHA-256 mismatch/, + ); + const escaped = structuredClone(manifest); + escaped.files.find((item) => item.role === "model_graph").path = "%2e%2e/escape.onnx"; + await assert.rejects( + onnx.createOnnxWebBackend({ + manifest: escaped, + baseUrl: "https://example.test/bundle/", + fetch: async () => new Response(new Uint8Array(), { status: 200 }), + }), + /bundle asset path must be relative/, + ); +}); + +test("malformed manifests are rejected before provider selection", async () => { + await assert.rejects( + onnx.createOnnxWebBackend({ manifest: { schema_version: "vons.bundle.manifest/v1" }, provider: "webgpu" }), + /metadata.backend|baseUrl|manifestUrl/, + ); +}); diff --git a/sdk/typescript/test/parity.test.mjs b/sdk/typescript/test/parity.test.mjs new file mode 100644 index 0000000000000000000000000000000000000000..c592c57d6ced7ceebbaf4c9e965b67fee0ccc1e9 --- /dev/null +++ b/sdk/typescript/test/parity.test.mjs @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +const { buildParityFixtures } = await import("../demo/parity.ts"); +const { validateRequest } = await import("../src/index.ts"); + +test("parity matrix covers candidate counts, question types, and explicit overflow", () => { + const fixtures = buildParityFixtures(); + assert.ok(fixtures.length >= 20); + assert.deepEqual(new Set(fixtures.map((item) => item.matrix.candidate_count)), new Set([2, 4, 8, 16, 32])); + assert.ok(fixtures.some((item) => item.request.questions[0]?.type === "boolean")); + assert.ok(fixtures.some((item) => item.request.questions[0]?.type === "score")); + assert.ok(fixtures.some((item) => item.expectedFailure)); + assert.equal(new Set(fixtures.map((item) => item.id)).size, fixtures.length); + for (const fixture of fixtures.filter((item) => !item.expectedFailure)) { + assert.doesNotThrow(() => validateRequest(fixture.request), fixture.id); + } +}); diff --git a/tests/test_browser_benchmark.py b/tests/test_browser_benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..6c09242462fe3b2a476184e99e340a7465a10661 --- /dev/null +++ b/tests/test_browser_benchmark.py @@ -0,0 +1,161 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +def _load_tool(): + path = Path(__file__).parents[1] / "tools" / "verify_browser_benchmark.py" + spec = importlib.util.spec_from_file_location("vons_browser_benchmark", path) + if spec is None or spec.loader is None: + raise RuntimeError("could not load browser benchmark verifier") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +benchmark_tool = _load_tool() + + +def _sample(repeat: int, status: str = "ok") -> dict[str, object]: + values = { + "tokenizeMs": 1.0 + repeat, + "prepareFeedMs": 2.0 + repeat, + "inferenceAndReadbackMs": 3.0 + repeat, + "postprocessMs": 4.0 + repeat, + "totalRequestMs": 10.0 + repeat, + } + if status == "error": + values = {key: None for key in values} + return { + **values, + "session_start": 1, + "repeat": repeat, + "run_id": "run-1", + "case_id": f"case-{repeat + 1}", + "backend": "direct", + "requested_provider": "wasm", + "observed_provider": None, + "observed_provider_status": "session_created; execution_partition_not_exposed", + "live_candidates": 4 if status == "ok" else None, + "allocated_candidates": 4 if status == "ok" else None, + "live_tokens": 12 if status == "ok" else None, + "sequence_length": 512 if status == "ok" else None, + "bundle_hash": "a" * 64, + "tokenizer_hash": "b" * 64, + "input_hash": "c" * 64, + "noise_hash": None, + "status": status, + "js_heap_used_bytes_before": None, + "js_heap_used_bytes_after": None, + } + + +class BrowserBenchmarkTests(unittest.TestCase): + def test_recomputes_raw_samples_and_allows_unavailable_memory(self) -> None: + report = { + "schema": "vons.browser-benchmark/v1", + "sessions": 1, + "warmup_excluded_per_session": 1, + "repeats_per_session": 3, + "run_id": "run-1", + "backend": "direct", + "cases": 1, + "requested_provider": "wasm", + "environment": {"user_agent": "test", "cross_origin_isolated": False}, + "runtime_info": None, + "execution_provider_evidence": "test evidence", + "session_records": [{"session_start": 1, "load_ms": 5.0, "status": "ok"}], + "samples": [_sample(0), _sample(1), _sample(2, "error")], + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "report.json" + path.write_text(json.dumps(report), encoding="utf-8") + result = benchmark_tool.verify_report(path) + + self.assertEqual(result["sample_rows"], 3) + self.assertEqual(result["successful_samples"], 2) + self.assertEqual(result["failed_samples"], 1) + self.assertEqual(result["recomputed_summary"]["total_request_ms_p50"], 10.5) + self.assertIsNone(result["recomputed_summary"]["js_heap_before_max"]) + + def test_rejects_failed_sample_with_fabricated_timing(self) -> None: + report = { + "schema": "vons.browser-benchmark/v1", + "sessions": 1, + "warmup_excluded_per_session": 1, + "repeats_per_session": 1, + "run_id": "run-1", + "backend": "direct", + "cases": 1, + "requested_provider": "webgpu", + "environment": {"user_agent": "test", "cross_origin_isolated": False}, + "runtime_info": None, + "execution_provider_evidence": "test evidence", + "session_records": [{"session_start": 1, "load_ms": 5.0, "status": "ok"}], + "samples": [_sample(0, "error")], + } + report["samples"][0]["requested_provider"] = "webgpu" + report["samples"][0]["totalRequestMs"] = 2.0 + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "report.json" + path.write_text(json.dumps(report), encoding="utf-8") + with self.assertRaises(ValueError): + benchmark_tool.verify_report(path) + + def test_rejects_sample_without_provenance_hashes(self) -> None: + report = { + "schema": "vons.browser-benchmark/v1", + "sessions": 1, + "warmup_excluded_per_session": 1, + "repeats_per_session": 1, + "run_id": "run-1", + "backend": "direct", + "cases": 1, + "requested_provider": "wasm", + "environment": {"user_agent": "test", "cross_origin_isolated": False}, + "runtime_info": None, + "execution_provider_evidence": "test evidence", + "session_records": [{"session_start": 1, "load_ms": 5.0, "status": "ok"}], + "samples": [_sample(0)], + } + report["samples"][0]["input_hash"] = None + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "report.json" + path.write_text(json.dumps(report), encoding="utf-8") + with self.assertRaises(ValueError): + benchmark_tool.verify_report(path) + + def test_rejects_partial_session_matrix_by_default(self) -> None: + report = { + "schema": "vons.browser-benchmark/v1", + "sessions": 2, + "warmup_excluded_per_session": 1, + "repeats_per_session": 1, + "run_id": "run-1", + "backend": "direct", + "cases": 1, + "requested_provider": "wasm", + "environment": {"user_agent": "test", "cross_origin_isolated": False}, + "runtime_info": None, + "execution_provider_evidence": "test evidence", + "session_records": [ + {"session_start": 1, "load_ms": 5.0, "status": "ok"}, + {"session_start": 2, "load_ms": 1.0, "status": "error", "error_kind": "fetch"}, + ], + "samples": [_sample(0)], + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "report.json" + path.write_text(json.dumps(report), encoding="utf-8") + with self.assertRaises(ValueError): + benchmark_tool.verify_report(path) + result = benchmark_tool.verify_report(path, allow_incomplete=True) + + self.assertFalse(result["complete"]) + self.assertTrue(result["incomplete_reasons"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_browser_benchmark_summary.py b/tests/test_browser_benchmark_summary.py new file mode 100644 index 0000000000000000000000000000000000000000..783cb531a522d13703a92f22fa4981dd04674e31 --- /dev/null +++ b/tests/test_browser_benchmark_summary.py @@ -0,0 +1,1130 @@ +"""Tests for tools/summarize_browser_benchmark.py. + +Covers: +- Exact expected linearly interpolated percentiles (p50, p95) for known datasets +- Null/failed/timeout row preservation with nullable timing slots +- Malformed rows, non-finite timings, wrong types, bad SHA256 +- Homogeneous condition enforcement and mixed-condition rejection +- Duplicate (run_id, session_start, repeat) identity rejection +- Insufficient-observation completeness flag without error +- CLI behavior via subprocess on a small valid fixture +""" + +from __future__ import annotations + +import json +import math +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +TOOLS_DIR = Path(__file__).resolve().parent.parent / "tools" +SUMMARY_TOOL = TOOLS_DIR / "summarize_browser_benchmark.py" + +sys.path.insert(0, str(TOOLS_DIR)) +from summarize_browser_benchmark import ( # noqa: E402, RUF100 + TIMING_FIELDS, + _validate_row, + percentile, + validate_and_aggregate, + validate_and_aggregate_report, +) + +SHA_A = "a" * 64 +SHA_B = "b" * 64 +SHA_C = "c" * 64 +SHA_D = "d" * 64 +SHA_NOISE = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +BASE_ENV = { + "user_agent": "TestBrowser/1.0", + "cross_origin_isolated": True, + "platform": "Test OS", +} + + +def _provenance(*, warmup: int = 10, semantics: str = "page_reload") -> dict[str, Any]: + return { + "environment": dict(BASE_ENV), + "execution_provider_evidence": "window.ort.env.backend='wasm'", + "warmup_excluded_per_session": warmup, + "session_start_semantics": semantics, + } + + +def _row( + *, + session_start: int = 1, + repeat: int = 0, + case_id: str = "case-0", + backend: str = "direct", + status: str = "ok", + error: str | None = None, + timings: dict[str, float] | None = None, + candidate_count: int = 4, + padded_option_slots: int = 8, + sequence_length: int = 64, + seed: int = 42, + noise_hash: str | None = None, + heap_before: float | None = None, + heap_after: float | None = None, + observed_provider: str = "wasm", +) -> dict[str, Any]: + if backend == "diffusion" and noise_hash is None: + noise_hash = SHA_NOISE + ok_timings: dict[str, float] = ( + timings + if timings is not None + else { + "tokenizeMs": 1.0, + "prepareFeedMs": 0.5, + "inferenceAndReadbackMs": 10.0, + "postprocessMs": 0.25, + "totalRequestMs": 11.75, + } + ) + row: dict[str, Any] = { + "run_id": "run-alpha", + "session_start": session_start, + "repeat": repeat, + "case_id": case_id, + "backend": backend, + "requested_provider": "wasm", + "observed_provider": observed_provider, + "bundle_hash": SHA_A, + "tokenizer_hash": SHA_B, + "input_hash": SHA_C, + "noise_hash": noise_hash, + "candidate_count": candidate_count, + "padded_option_slots": padded_option_slots, + "sequence_length": sequence_length, + "seed": seed, + "ort_version": "1.20.1", + "status": status, + "error": error, + "provenance": _provenance(), + } + if status == "ok": + row.update(ok_timings) + if heap_before is not None: + row["js_heap_used_bytes_before"] = heap_before + if heap_after is not None: + row["js_heap_used_bytes_after"] = heap_after + else: + for field in TIMING_FIELDS: + row[field] = None + if timings is not None: + for field, value in timings.items(): + row[field] = value + if error is None: + row["error"] = "simulated failure" + return row + + +def _write_jsonl(tmp_path: Path, rows: list[dict[str, Any]], *, name: str = "input.jsonl") -> Path: + path = tmp_path / name + with path.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, sort_keys=True) + "\n") + return path + + +# --------------------------------------------------------------------------- +# percentile linear interpolation correctness +# --------------------------------------------------------------------------- + + +class TestPercentile: + def test_empty_is_null(self) -> None: + assert percentile([], 0.5) is None + + def test_single_value(self) -> None: + assert percentile([7.0], 0.5) == 7.0 + + def test_two_values_p50_is_midpoint(self) -> None: + # index = (2-1)*0.5 = 0.5 -> lower=0, upper=1 -> midpoint + assert percentile([10.0, 20.0], 0.5) == pytest.approx(15.0) + + def test_two_values_p95_near_upper(self) -> None: + # index = 1 * 0.95 = 0.95 -> 10 + 10*0.95 = 19.5 + assert percentile([10.0, 20.0], 0.95) == pytest.approx(19.5) + + def test_four_values_p50_exact_interpolation(self) -> None: + # sorted: [1,2,3,4]; index = (4-1)*0.5 = 1.5 + # lower=1 -> 2, upper=2 -> 3, fraction=0.5 -> 2 + 1*0.5 = 2.5 + assert percentile([3.0, 1.0, 4.0, 2.0], 0.5) == pytest.approx(2.5) + + def test_eight_values_p95(self) -> None: + values = [float(x) for x in range(1, 9)] + # sorted: [1..8]; index = (8-1)*0.95 = 7*0.95 = 6.65 + # lower=6 -> 7.0, upper=7 -> 8.0, delta=0.65 -> 7.0 + 1.0*0.65 = 7.65 + assert percentile(values, 0.95) == pytest.approx(7.65) + + def test_fraction_out_of_range_rejected(self) -> None: + with pytest.raises(ValueError): + percentile([1.0, 2.0], 1.5) + with pytest.raises(ValueError): + percentile([1.0, 2.0], -0.1) + + +# --------------------------------------------------------------------------- +# Row validation +# --------------------------------------------------------------------------- + + +class TestRowValidation: + def test_missing_status_rejected(self) -> None: + row = _row() + del row["status"] + with pytest.raises(ValueError, match="missing required field status"): + _validate_row(row, 1) + + def test_invalid_status_string_rejected(self) -> None: + row = _row(status="pending") + with pytest.raises(ValueError, match="status must be ok, error, or timeout"): + _validate_row(row, 1) + + def test_ok_row_with_error_value_rejected(self) -> None: + row = _row(status="ok", error="should be empty") + with pytest.raises(ValueError, match="error must be null for successful"): + _validate_row(row, 1) + + def test_failed_row_with_partial_timings_allowed(self) -> None: + row = _row(status="error", timings={"totalRequestMs": 500.0, "tokenizeMs": 10.0}) + assert row["status"] == "error" + assert row["totalRequestMs"] == 500.0 + assert row["tokenizeMs"] == 10.0 + assert row["prepareFeedMs"] is None + _validate_row(row, 1) + + def test_successful_row_missing_timing_rejected(self) -> None: + row = _row() + del row["inferenceAndReadbackMs"] + with pytest.raises(ValueError, match="successful sample requires inferenceAndReadbackMs"): + _validate_row(row, 1) + + def test_nonfinite_timing_rejected(self) -> None: + row = _row(timings={"tokenizeMs": math.nan, "prepareFeedMs": 0.5, + "inferenceAndReadbackMs": 10.0, "postprocessMs": 0.25, + "totalRequestMs": 11.75}) + with pytest.raises((ValueError, TypeError), match="finite"): + _validate_row(row, 1) + + def test_negative_timing_rejected(self) -> None: + row = _row(timings={"tokenizeMs": -1.0, "prepareFeedMs": 0.5, + "inferenceAndReadbackMs": 10.0, "postprocessMs": 0.25, + "totalRequestMs": 11.75}) + with pytest.raises(ValueError, match="finite non-negative"): + _validate_row(row, 1) + + def test_invalid_sha256_rejected(self) -> None: + row = _row() + row["bundle_hash"] = "short-and-wrong" + with pytest.raises(ValueError, match="SHA-256 hex string"): + _validate_row(row, 1) + + def test_uppercase_sha256_rejected(self) -> None: + row = _row() + row["bundle_hash"] = "A" * 64 + with pytest.raises(ValueError, match="lowercase SHA-256"): + _validate_row(row, 1) + + def test_non_positive_shape_rejected(self) -> None: + row = _row(candidate_count=0) + with pytest.raises(ValueError, match="positive integer"): + _validate_row(row, 1) + row2 = _row(padded_option_slots=-5) + with pytest.raises(ValueError, match="positive integer"): + _validate_row(row2, 2) + row3 = _row(sequence_length=0) + with pytest.raises(ValueError, match="positive integer"): + _validate_row(row3, 3) + + def test_diffusion_noise_required(self) -> None: + row = _row(backend="diffusion", noise_hash=None) + row["noise_hash"] = None + with pytest.raises(ValueError, match="noise_hash is required for diffusion"): + _validate_row(row, 1) + + def test_repeat_negative_rejected(self) -> None: + row = _row(repeat=-1) + with pytest.raises(ValueError, match="repeat must be a non-negative integer"): + _validate_row(row, 1) + + def test_requested_provider_must_be_wasm_or_webgpu(self) -> None: + row = _row() + row["requested_provider"] = "cpu" + with pytest.raises(ValueError, match="requested_provider must be wasm or webgpu"): + _validate_row(row, 1) + + +# --------------------------------------------------------------------------- +# End-to-end aggregation with actual expected percentiles +# --------------------------------------------------------------------------- + + +def _build_120row_fixture( + *, + backend: str = "direct", + extra_failures: int = 0, + warmup: int = 10, +) -> list[dict[str, Any]]: + """10 sessions x 12 repeats = 120 rows with deterministic increasing timings. + + totalRequestMs values for successful rows = 1.0, 2.0, ..., N_success.0. + p50 (120 successes, all ok): + index = (120-1)*0.5 = 59.5 -> values[59]=60.0, values[60]=61.0, frac=0.5 -> 60.5 + p95: + index = 119*0.95 = 113.05 -> values[113]=114.0, values[114]=115.0, frac=0.05 + -> 114.0 + 1.0*0.05 = 114.05 + """ + rows: list[dict[str, Any]] = [] + counter = 0 + for session in range(1, 11): + for repeat in range(12): + counter += 1 + if extra_failures > 0 and counter % 15 == 0: + rows.append( + _row( + session_start=session, + repeat=repeat, + case_id=f"case-{counter}", + backend=backend, + status="error", + error="simulated", + ) + ) + extra_failures -= 1 + else: + total = float(counter) + tokenize = 0.1 * counter + prepare = 0.05 * counter + inference = 0.8 * counter + post = 0.05 * counter + rows.append( + _row( + session_start=session, + repeat=repeat, + case_id=f"case-{counter}", + backend=backend, + status="ok", + timings={ + "tokenizeMs": tokenize, + "prepareFeedMs": prepare, + "inferenceAndReadbackMs": inference, + "postprocessMs": post, + "totalRequestMs": total, + }, + heap_before=1000.0 + counter, + heap_after=2000.0 + counter, + ) + ) + rows[-1]["provenance"] = _provenance(warmup=warmup) + return rows + + +class TestAggregationPercentiles: + def test_120rows_percentiles_match_exact_expected(self, tmp_path: Path) -> None: + rows = _build_120row_fixture() + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + + assert summary["sample_counts"]["successful"] == 120 + assert summary["sample_counts"]["failed"] == 0 + assert summary["sample_counts"]["unique_session_starts"] == 10 + assert summary["measurement_completeness"]["complete"] is True + + p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] + p95 = summary["stage_percentiles_ms"]["totalRequestMs"]["p95"] + assert p50 == pytest.approx(60.5) + assert p95 == pytest.approx(114.05) + + # tokenizeMs values = 0.1 * counter for counter=1..120 + # sorted values are 0.1, 0.2, ..., 12.0 + # p50: index=59.5 -> 6.0 + (6.1-6.0)*0.5 = 6.05 + assert summary["stage_percentiles_ms"]["tokenizeMs"]["p50"] == pytest.approx(6.05) + # p95: index=113.05 -> values[113]=11.4, values[114]=11.5, frac=0.05 -> 11.405 + assert summary["stage_percentiles_ms"]["tokenizeMs"]["p95"] == pytest.approx(11.405) + + def test_failed_rows_counted_timings_nullable(self, tmp_path: Path) -> None: + rows = _build_120row_fixture(extra_failures=8) + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + + assert summary["sample_counts"]["successful"] == 112 + assert summary["sample_counts"]["failed"] == 8 + assert summary["measurement_completeness"]["complete"] is True + + # 112 successes: totalRequestMs are the 112 counter values that + # are not divisible by 15. Counters divisible by 15 (15,30,...,105,120) + # are failures. Total counters divisible by 15 up to 120: 120//15=8. + # totalRequestMs values are 1..14,16..29,...,106..119 (8 missing + # values: 15,30,45,60,75,90,105,120). All increasing order is still + # preserved because values increase monotonically with counter and + # only the multiples of 15 are removed. + # + # For percentile of 112 values: + # p50 index = (112-1)*0.5 = 55.5 -> values[55] and values[56] + # Before each multiple-of-15 threshold, 1 value has been skipped per + # prior group. Which values are at index 55/56? + # For each group of 15 counters, 14 survive. Group 1 (1-14) -> indices 0..13 (14). + # Group 2 (16-29) -> indices 14..27 (14, cumulative 28). + # Group 3 (31-44) -> indices 28..41 (14, cumulative 42). + # Group 4 (46-59) -> indices 42..55 (14, cumulative 56). + # So index 55 = last of group 4 -> counter 59 (value 59.0). + # Index 56 = first of group 5 -> counter 61 (value 61.0). + # Interpolation at 55.5 -> 0.5 fraction: 59.0 + 2.0 * 0.5 = 60.0 + p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] + assert p50 == pytest.approx(60.0) + + def test_heap_maxima_from_observations_only(self, tmp_path: Path) -> None: + rows = _build_120row_fixture() + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + # counter 1..120 -> heap_before = 1000 + counter -> max at 1120 + # counter 1..120 -> heap_after = 2000 + counter -> max at 2120 + assert summary["heap_maxima_bytes"]["js_heap_used_bytes_before"] == pytest.approx(1120.0) + assert summary["heap_maxima_bytes"]["js_heap_used_bytes_after"] == pytest.approx(2120.0) + + def test_all_null_heap_yields_null_maxima(self, tmp_path: Path) -> None: + rows = _build_120row_fixture() + for row in rows: + if row["status"] == "ok": + row.pop("js_heap_used_bytes_before", None) + row.pop("js_heap_used_bytes_after", None) + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["heap_maxima_bytes"]["js_heap_used_bytes_before"] is None + assert summary["heap_maxima_bytes"]["js_heap_used_bytes_after"] is None + + +# --------------------------------------------------------------------------- +# Empty input, duplicates, mixed conditions +# --------------------------------------------------------------------------- + + +class TestValidationFailuresAggregate: + def test_empty_input_raises(self, tmp_path: Path) -> None: + path = tmp_path / "empty.jsonl" + path.write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="no samples"): + validate_and_aggregate(path) + + def test_duplicate_identity_raises(self, tmp_path: Path) -> None: + rows = [ + _row(session_start=1, repeat=0, case_id="a"), + _row(session_start=1, repeat=0, case_id="b"), + ] + input_path = _write_jsonl(tmp_path, rows) + with pytest.raises(ValueError, match="duplicate.*identity"): + validate_and_aggregate(input_path) + + def test_mixed_bundle_hash_raises(self, tmp_path: Path) -> None: + rows = [_row(session_start=1, repeat=0), _row(session_start=1, repeat=1)] + rows[1]["bundle_hash"] = SHA_D + input_path = _write_jsonl(tmp_path, rows) + with pytest.raises(ValueError, match="mixed experimental conditions"): + validate_and_aggregate(input_path) + + def test_mixed_backend_raises(self, tmp_path: Path) -> None: + rows = [ + _row(session_start=1, repeat=0, backend="direct"), + _row(session_start=1, repeat=1, backend="diffusion", noise_hash=SHA_NOISE), + ] + input_path = _write_jsonl(tmp_path, rows) + with pytest.raises(ValueError, match="mixed experimental conditions"): + validate_and_aggregate(input_path) + + def test_mixed_environment_raises(self, tmp_path: Path) -> None: + rows = [_row(session_start=1, repeat=0), _row(session_start=1, repeat=1)] + rows[1]["provenance"] = _provenance() + rows[1]["provenance"]["environment"]["platform"] = "Different OS" + input_path = _write_jsonl(tmp_path, rows) + with pytest.raises(ValueError, match="mixed experimental conditions"): + validate_and_aggregate(input_path) + + def test_mixed_warmup_raises(self, tmp_path: Path) -> None: + rows = [_row(session_start=1, repeat=0), _row(session_start=1, repeat=1)] + rows[1]["provenance"] = _provenance(warmup=20) + input_path = _write_jsonl(tmp_path, rows) + with pytest.raises(ValueError, match="mixed experimental conditions"): + validate_and_aggregate(input_path) + + def test_diffusion_backend_homogeneous(self, tmp_path: Path) -> None: + """Diffusion samples must carry noise_hash; varying noise_hash by row is allowed.""" + rows = [] + for session in range(1, 7): + for repeat in range(20): + idx = session * 100 + repeat + rows.append( + _row( + session_start=session, + repeat=repeat, + case_id=f"case-{idx}", + backend="diffusion", + noise_hash=f"{idx:x}".zfill(64), + ) + ) + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["experimental_condition"]["backend"] == "diffusion" + assert summary["sample_counts"]["successful"] == 120 + assert summary["sample_counts"]["failed"] == 0 + + +# --------------------------------------------------------------------------- +# Completeness flag and provenance semantics +# --------------------------------------------------------------------------- + + +class TestCompletenessAndProvenance: + def test_insufficient_sessions_reported_incomplete(self, tmp_path: Path) -> None: + rows = [] + for session in range(1, 4): # only 3 sessions + for repeat in range(50): + rows.append( + _row( + session_start=session, + repeat=repeat, + case_id=f"case-{session}-{repeat}", + ) + ) + # warmup per session = 10, rows = 150 -> two thresholds satisfied, + # but session count < 5 + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + comp = summary["measurement_completeness"] + assert comp["complete"] is False + assert comp["actual_session_starts"] == 3 + reasons = " ".join(comp["incomplete_reasons"]) + assert "independent session starts" in reasons + + def test_insufficient_warmup_reported_incomplete(self, tmp_path: Path) -> None: + rows = _build_120row_fixture(warmup=5) + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + comp = summary["measurement_completeness"] + assert comp["complete"] is False + assert comp["actual_warmup_excluded_per_session"] == 5 + + def test_120rows_10sessions_is_complete(self, tmp_path: Path) -> None: + rows = _build_120row_fixture() + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["measurement_completeness"]["complete"] is True + assert summary["measurement_completeness"]["incomplete_reasons"] == [] + + def test_page_semantics_not_process_cold_claimable(self, tmp_path: Path) -> None: + rows = [_row(session_start=1, repeat=0)] + rows[0]["provenance"]["session_start_semantics"] = "in_page_repeat" + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["process_cold_claimable"] is False + + def test_process_restart_semantics_still_false_v1(self, tmp_path: Path) -> None: + rows = [_row(session_start=1, repeat=0)] + rows[0]["provenance"]["session_start_semantics"] = "process_restart" + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["process_cold_claimable"] is False + assert "always False regardless of semantics string" in summary["process_cold_note"] + + def test_requested_provider_match_still_false_v1(self, tmp_path: Path) -> None: + rows = [ + _row(session_start=1, repeat=0, observed_provider="wasm"), + _row(session_start=1, repeat=1, observed_provider="wasm"), + ] + rows[0]["provenance"]["execution_provider_evidence"] = "window.ort.env.backend='wasm'" + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["requested_is_proven"] is False + assert summary["experimental_condition"]["observed_provider"] == "wasm" + assert summary["experimental_condition"]["provenance"]["execution_provider_evidence"] == "window.ort.env.backend='wasm'" + assert "requested_is_proven is always False in v1" in summary["provider_claim_note"] + + def test_observed_provider_mismatch_not_proven(self, tmp_path: Path) -> None: + rows = [_row(session_start=1, repeat=0, observed_provider=None)] + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["requested_is_proven"] is False + + +# --------------------------------------------------------------------------- +# CLI integration +# --------------------------------------------------------------------------- + + +class TestCLI: + def test_cli_writes_summary(self, tmp_path: Path) -> None: + rows = _build_120row_fixture() + input_path = _write_jsonl(tmp_path, rows) + output_path = tmp_path / "summary.json" + result = subprocess.run( + [ + sys.executable, + str(SUMMARY_TOOL), + "--input", + str(input_path), + "--output", + str(output_path), + ], + capture_output=True, + check=False, + text=True, + ) + assert result.returncode == 0, result.stderr + assert output_path.exists() + loaded = json.loads(output_path.read_text(encoding="utf-8")) + assert loaded["schema"] == "vons.browser-benchmark-summary/v1" + assert loaded["sample_counts"]["successful"] == 120 + assert loaded["stage_percentiles_ms"]["totalRequestMs"]["p50"] == pytest.approx(60.5) + + def test_cli_validation_fails_before_writing(self, tmp_path: Path) -> None: + rows = [ + _row(session_start=1, repeat=0), + _row(session_start=1, repeat=0, case_id="dup"), + ] + input_path = _write_jsonl(tmp_path, rows) + output_path = tmp_path / "summary.json" + result = subprocess.run( + [ + sys.executable, + str(SUMMARY_TOOL), + "--input", + str(input_path), + "--output", + str(output_path), + ], + capture_output=True, + check=False, + text=True, + ) + assert result.returncode != 0 + assert not output_path.exists() or output_path.stat().st_size == 0 + + +# --------------------------------------------------------------------------- +# Error rows preserve nullable timing slots at summary level (not erased) +# --------------------------------------------------------------------------- + + +class TestErrorRowPreservation: + def test_timeout_rows_counted_as_failed_with_nullable_slots(self, tmp_path: Path) -> None: + rows = [ + _row(session_start=1, repeat=0, status="timeout", error="gpu hang"), + _row(session_start=1, repeat=1, status="ok"), + ] + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["sample_counts"]["failed"] == 1 + assert summary["sample_counts"]["successful"] == 1 + # successful timings are only row[1]; all percentiles equal that row's values + assert summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] == pytest.approx(11.75) + + +# --------------------------------------------------------------------------- +# Integrator regressions: v1 hard-pessimism, seed 0, partial error timings, +# shape bounds +# --------------------------------------------------------------------------- + + +class TestIntegratorRegressions: + def test_unknown_semantics_still_false_process_cold(self, tmp_path: Path) -> None: + rows = [_row(session_start=1, repeat=0)] + rows[0]["provenance"]["session_start_semantics"] = "totally_unknown_semantic_XYZ" + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["process_cold_claimable"] is False + assert summary["experimental_condition"]["provenance"]["session_start_semantics"] == "totally_unknown_semantic_XYZ" + assert "always False regardless of semantics string" in summary["process_cold_note"] + + def test_matching_provider_unverified_evidence_not_proven(self, tmp_path: Path) -> None: + rows = [_row(session_start=1, repeat=0, observed_provider="wasm")] + rows[0]["requested_provider"] = "wasm" + rows[0]["observed_provider"] = "wasm" + rows[0]["provenance"]["execution_provider_evidence"] = "raw browser string: wasm observed" + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["requested_is_proven"] is False + assert summary["experimental_condition"]["requested_provider"] == "wasm" + assert summary["experimental_condition"]["observed_provider"] == "wasm" + assert summary["experimental_condition"]["provenance"]["execution_provider_evidence"] == "raw browser string: wasm observed" + assert "no structured verified execution artifact" in summary["provider_claim_note"] + + def test_seed_zero_is_valid(self, tmp_path: Path) -> None: + rows = _build_120row_fixture() + for row in rows: + row["seed"] = 0 + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["sample_counts"]["successful"] == 120 + assert summary["experimental_condition"]["candidate_count"] == 4 + p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] + assert p50 == pytest.approx(60.5) + + def test_partial_measured_error_row_validated_and_excluded_from_percentiles(self, tmp_path: Path) -> None: + rows = _build_120row_fixture() + error_row = _row( + session_start=11, + repeat=0, + case_id="case-extra-error", + status="error", + error="inference timeout after prepare", + timings={ + "tokenizeMs": 12.0, + "prepareFeedMs": 5.5, + "totalRequestMs": 500.0, + }, + ) + rows.append(error_row) + input_path = _write_jsonl(tmp_path, rows) + summary = validate_and_aggregate(input_path) + assert summary["sample_counts"]["successful"] == 120 + assert summary["sample_counts"]["failed"] == 1 + p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] + p95 = summary["stage_percentiles_ms"]["totalRequestMs"]["p95"] + assert p50 == pytest.approx(60.5) + assert p95 == pytest.approx(114.05) + + def test_shape_candidate_exceeds_padded_rejected(self) -> None: + row = _row(candidate_count=10, padded_option_slots=8) + with pytest.raises(ValueError, match="candidate_count must be <= padded_option_slots"): + _validate_row(row, 1) + + def test_shape_padded_exceeds_32_rejected(self) -> None: + row = _row(candidate_count=16, padded_option_slots=33) + with pytest.raises(ValueError, match="padded_option_slots must be <= 32"): + _validate_row(row, 1) + + def test_shape_sequence_exceeds_512_rejected(self) -> None: + row = _row(sequence_length=513) + with pytest.raises(ValueError, match="sequence_length must be <= 512"): + _validate_row(row, 1) + + def test_shape_boundary_values_accepted(self) -> None: + row = _row(candidate_count=32, padded_option_slots=32, sequence_length=512) + _validate_row(row, 1) + + +# --------------------------------------------------------------------------- +# --report-input mode: normalize live_candidates/allocated_candidates/ +# sequence_tokens from existing v1 browser-benchmark JSON report, preserve +# top-level provenance, and never invent missing values +# --------------------------------------------------------------------------- + + +def _build_v1_report_fixture( + *, + backend: str = "direct", + live_field_name: str = "live_candidates", + allocated_field_name: str = "allocated_candidates", + seq_field_name: str = "sequence_length", + top_level_ort: str | None = "1.20.1", + include_session_start_semantics: bool = True, + include_environment: bool = True, + extra_failures: int = 0, +) -> dict[str, Any]: + samples: list[dict[str, Any]] = [] + counter = 0 + for session in range(1, 11): + for repeat in range(12): + counter += 1 + if extra_failures > 0 and counter % 15 == 0: + sample: dict[str, Any] = { + "run_id": "run-alpha", + "session_start": session, + "repeat": repeat, + "case_id": f"case-{counter}", + "backend": backend, + "requested_provider": "wasm", + "observed_provider": "wasm", + "bundle_hash": SHA_A, + "tokenizer_hash": SHA_B, + "input_hash": SHA_C, + "noise_hash": None, + live_field_name: 4, + allocated_field_name: 8, + seq_field_name: 64, + "seed": 42, + "ort_version": top_level_ort, + "status": "error", + "error": "simulated", + "tokenizeMs": None, + "prepareFeedMs": None, + "inferenceAndReadbackMs": None, + "postprocessMs": None, + "totalRequestMs": None, + } + if backend == "diffusion": + sample["noise_hash"] = SHA_NOISE + samples.append(sample) + extra_failures -= 1 + else: + total = float(counter) + tokenize = 0.1 * counter + prepare = 0.05 * counter + inference = 0.8 * counter + post = 0.05 * counter + sample = { + "run_id": "run-alpha", + "session_start": session, + "repeat": repeat, + "case_id": f"case-{counter}", + "backend": backend, + "requested_provider": "wasm", + "observed_provider": "wasm", + "bundle_hash": SHA_A, + "tokenizer_hash": SHA_B, + "input_hash": SHA_C, + "noise_hash": None, + live_field_name: 4, + allocated_field_name: 8, + seq_field_name: 64, + "seed": 42, + "ort_version": top_level_ort, + "status": "ok", + "error": None, + "tokenizeMs": tokenize, + "prepareFeedMs": prepare, + "inferenceAndReadbackMs": inference, + "postprocessMs": post, + "totalRequestMs": total, + "js_heap_used_bytes_before": 1000.0 + counter, + "js_heap_used_bytes_after": 2000.0 + counter, + } + if backend == "diffusion": + sample["noise_hash"] = SHA_NOISE + samples.append(sample) + report: dict[str, Any] = { + "schema": "vons.browser-benchmark/v1", + "run_id": "run-alpha", + "backend": backend, + "requested_provider": "wasm", + "warmup_excluded_per_session": 10, + "samples": samples, + } + if include_environment: + report["environment"] = dict(BASE_ENV) + report["execution_provider_evidence"] = "window.ort.env.backend='wasm'" + if include_session_start_semantics: + report["session_start_semantics"] = "page_reload" + if top_level_ort is not None: + report["ort_version"] = top_level_ort + return report + + +def _write_report(tmp_path: Path, report: dict[str, Any], *, name: str = "report.json") -> Path: + path = tmp_path / name + path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") + return path + + +class TestReportInputNormalization: + def test_report_happy_path_matches_jsonl_percentiles(self, tmp_path: Path) -> None: + report = _build_v1_report_fixture() + report_path = _write_report(tmp_path, report) + summary = validate_and_aggregate_report(report_path) + assert summary["sample_counts"]["successful"] == 120 + assert summary["sample_counts"]["failed"] == 0 + assert summary["measurement_completeness"]["complete"] is True + p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] + p95 = summary["stage_percentiles_ms"]["totalRequestMs"]["p95"] + assert p50 == pytest.approx(60.5) + assert p95 == pytest.approx(114.05) + assert summary["experimental_condition"]["provenance"]["session_start_semantics"] == "page_reload" + assert summary["process_cold_claimable"] is False + assert summary["requested_is_proven"] is False + + def test_report_sequence_tokens_fallback_alias(self, tmp_path: Path) -> None: + report = _build_v1_report_fixture(seq_field_name="sequence_tokens") + for sample in report["samples"]: + assert "sequence_length" not in sample + assert sample["sequence_tokens"] == 64 + report_path = _write_report(tmp_path, report) + summary = validate_and_aggregate_report(report_path) + assert summary["experimental_condition"]["sequence_length"] == 64 + assert summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] == pytest.approx(60.5) + + def test_report_missing_live_candidates_rejected(self, tmp_path: Path) -> None: + report = _build_v1_report_fixture() + for sample in report["samples"]: + del sample["live_candidates"] + report_path = _write_report(tmp_path, report) + with pytest.raises(ValueError, match="missing live_candidates"): + validate_and_aggregate_report(report_path) + + def test_report_missing_allocated_candidates_rejected(self, tmp_path: Path) -> None: + report = _build_v1_report_fixture() + for sample in report["samples"]: + del sample["allocated_candidates"] + report_path = _write_report(tmp_path, report) + with pytest.raises(ValueError, match="missing allocated_candidates"): + validate_and_aggregate_report(report_path) + + def test_report_missing_sequence_both_names_rejected(self, tmp_path: Path) -> None: + report = _build_v1_report_fixture() + for sample in report["samples"]: + del sample["sequence_length"] + report_path = _write_report(tmp_path, report) + with pytest.raises(ValueError, match="missing sequence_length"): + validate_and_aggregate_report(report_path) + + def test_report_missing_session_start_semantics_rejected_no_invent(self, tmp_path: Path) -> None: + report = _build_v1_report_fixture(include_session_start_semantics=False) + assert "session_start_semantics" not in report + report_path = _write_report(tmp_path, report) + with pytest.raises(ValueError, match="do not invent missing values"): + validate_and_aggregate_report(report_path) + + def test_report_missing_ort_version_sample_and_top_rejected(self, tmp_path: Path) -> None: + report = _build_v1_report_fixture(top_level_ort=None) + for sample in report["samples"]: + del sample["ort_version"] + report_path = _write_report(tmp_path, report) + with pytest.raises(ValueError, match="missing ort_version"): + validate_and_aggregate_report(report_path) + + def test_report_top_level_ort_used_as_fallback(self, tmp_path: Path) -> None: + report = _build_v1_report_fixture(top_level_ort="1.21.0-custom") + for sample in report["samples"]: + del sample["ort_version"] + report_path = _write_report(tmp_path, report) + summary = validate_and_aggregate_report(report_path) + assert summary["experimental_condition"]["ort_version"] == "1.21.0-custom" + assert summary["sample_counts"]["successful"] == 120 + + def test_report_wrong_schema_rejected(self, tmp_path: Path) -> None: + report = _build_v1_report_fixture() + report["schema"] = "other.schema/v2" + report_path = _write_report(tmp_path, report) + with pytest.raises(ValueError, match="unexpected browser benchmark schema"): + validate_and_aggregate_report(report_path) + + +# --------------------------------------------------------------------------- +# CLI --report-input subprocess paths +# --------------------------------------------------------------------------- + + +class TestCLIReportInput: + def test_cli_report_input_writes_summary(self, tmp_path: Path) -> None: + report = _build_v1_report_fixture() + report_path = _write_report(tmp_path, report) + output_path = tmp_path / "summary.json" + result = subprocess.run( + [ + sys.executable, + str(SUMMARY_TOOL), + "--report-input", + str(report_path), + "--output", + str(output_path), + ], + capture_output=True, + check=False, + text=True, + ) + assert result.returncode == 0, result.stderr + assert output_path.exists() + loaded = json.loads(output_path.read_text(encoding="utf-8")) + assert loaded["schema"] == "vons.browser-benchmark-summary/v1" + assert loaded["sample_counts"]["successful"] == 120 + assert loaded["stage_percentiles_ms"]["totalRequestMs"]["p50"] == pytest.approx(60.5) + assert loaded["process_cold_claimable"] is False + assert loaded["requested_is_proven"] is False + + def test_cli_mutually_exclusive_inputs(self, tmp_path: Path) -> None: + rows = _build_120row_fixture() + input_path = _write_jsonl(tmp_path, rows) + report = _build_v1_report_fixture() + report_path = _write_report(tmp_path, report) + output_path = tmp_path / "summary.json" + result = subprocess.run( + [ + sys.executable, + str(SUMMARY_TOOL), + "--input", + str(input_path), + "--report-input", + str(report_path), + "--output", + str(output_path), + ], + capture_output=True, + check=False, + text=True, + ) + assert result.returncode != 0 + assert "not allowed with" in result.stderr or "argument --report-input: not allowed" in result.stderr + + +# --------------------------------------------------------------------------- +# Pass 3: Real report compatibility regressions — matches actual report +# shape in reports/browser-direct-{wasm,webgpu-controlled}-merged.json: +# - ort_version via top-level runtime_info.ortVersion (camelCase, nested) +# - seed absent in samples => null allowed for direct backend when +# noise_hash=null (deterministic-direct not_applicable) +# - observed_provider=null (harness doesn't expose execution partition) +# - heap keys js_heap_used_bytes_before/after canonical +# --------------------------------------------------------------------------- + + +def _build_real_direct_report( + *, + runtime_ort: str = "1.30.0", + top_ort_version_key_present: bool = False, + top_ort_value: str | None = None, + seed_present_in_samples: bool = False, + noise_hash_value: None | str = None, +) -> dict[str, Any]: + samples: list[dict[str, Any]] = [] + counter = 0 + for session in range(1, 7): + for repeat in range(20): + counter += 1 + total = float(counter) + sample: dict[str, Any] = { + "run_id": "run-real-shape", + "session_start": session, + "repeat": repeat, + "case_id": f"case-{counter}", + "backend": "direct", + "requested_provider": "wasm", + "observed_provider": None, + "observed_provider_status": "session_created; execution_partition_not_exposed", + "bundle_hash": SHA_A, + "tokenizer_hash": SHA_B, + "input_hash": SHA_C, + "noise_hash": noise_hash_value, + "live_candidates": 4, + "allocated_candidates": 8, + "sequence_length": 64, + "status": "ok", + "error": None, + "tokenizeMs": 0.1 * counter, + "prepareFeedMs": 0.05 * counter, + "inferenceAndReadbackMs": 0.8 * counter, + "postprocessMs": 0.05 * counter, + "totalRequestMs": total, + "js_heap_used_bytes_before": 1000.0 + counter, + "js_heap_used_bytes_after": 2000.0 + counter, + } + if seed_present_in_samples: + sample["seed"] = 42 + samples.append(sample) + report: dict[str, Any] = { + "schema": "vons.browser-benchmark/v1", + "run_id": "run-real-shape", + "backend": "direct", + "requested_provider": "wasm", + "environment": dict(BASE_ENV), + "execution_provider_evidence": "requested provider and session creation only", + "session_start_semantics": "one fresh page/session per source report; browser caches may persist", + "warmup_excluded_per_session": 10, + "runtime_info": { + "ortVersion": runtime_ort, + "provider": "wasm", + "wasmNumThreads": 1, + "webgpuAdapterRequested": False, + "webgpuDeviceRequested": False, + }, + "samples": samples, + } + if top_ort_version_key_present: + report["ort_version"] = top_ort_value + return report + + +class TestRealReportCompatibility: + def test_runtime_info_ortversion_used_as_top_fallback(self, tmp_path: Path) -> None: + report = _build_real_direct_report(runtime_ort="1.30.0-fromRuntimeInfo") + report_path = _write_report(tmp_path, report) + summary = validate_and_aggregate_report(report_path) + assert summary["experimental_condition"]["ort_version"] == "1.30.0-fromRuntimeInfo" + assert summary["sample_counts"]["successful"] == 120 + p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] + assert p50 == pytest.approx(60.5) + + def test_top_ort_version_preferred_over_runtime_info(self, tmp_path: Path) -> None: + report = _build_real_direct_report( + runtime_ort="1.30.0-runtime-should-not-be-used", + top_ort_version_key_present=True, + top_ort_value="1.31.0-topPreferred", + ) + report_path = _write_report(tmp_path, report) + summary = validate_and_aggregate_report(report_path) + assert summary["experimental_condition"]["ort_version"] == "1.31.0-topPreferred" + + def test_missing_both_ort_raises_mentions_runtimeinfo(self, tmp_path: Path) -> None: + report = _build_real_direct_report(runtime_ort="1.30.0") + del report["runtime_info"] + report_path = _write_report(tmp_path, report) + with pytest.raises(ValueError, match="runtime_info.ortVersion"): + validate_and_aggregate_report(report_path) + + def test_direct_samples_no_seed_noisehash_null_resolves_seed_null(self, tmp_path: Path) -> None: + report = _build_real_direct_report(seed_present_in_samples=False, noise_hash_value=None) + for sample in report["samples"]: + assert "seed" not in sample + assert sample["noise_hash"] is None + report_path = _write_report(tmp_path, report) + summary = validate_and_aggregate_report(report_path) + assert summary["experimental_condition"]["seed"] is None + assert summary["sample_counts"]["successful"] == 120 + p50 = summary["stage_percentiles_ms"]["totalRequestMs"]["p50"] + assert p50 == pytest.approx(60.5) + + def test_direct_with_noise_hash_still_requires_seed_not_null(self, tmp_path: Path) -> None: + report = _build_real_direct_report(seed_present_in_samples=False, noise_hash_value=SHA_NOISE) + for sample in report["samples"]: + assert "seed" not in sample + assert sample["noise_hash"] == SHA_NOISE + report_path = _write_report(tmp_path, report) + with pytest.raises(ValueError, match="missing seed"): + validate_and_aggregate_report(report_path) + + def test_diffusion_samples_require_seed_never_null(self) -> None: + row = _row(backend="diffusion", noise_hash=SHA_NOISE) + row["seed"] = None + with pytest.raises(ValueError, match="seed may only be null for direct backend"): + _validate_row(row, 1) + + def test_direct_with_seed_integer_still_accepted(self) -> None: + row = _row(seed=17) + row["noise_hash"] = None + row["seed"] = 17 + _validate_row(row, 1) + + def test_direct_seed_null_noise_hash_null_accepted_at_row_level(self) -> None: + row = _row(seed=42) + row["seed"] = None + row["noise_hash"] = None + _validate_row(row, 1) + + def test_observed_provider_null_preserved_in_summary(self, tmp_path: Path) -> None: + report = _build_real_direct_report() + report_path = _write_report(tmp_path, report) + summary = validate_and_aggregate_report(report_path) + assert summary["experimental_condition"]["observed_provider"] is None + assert summary["requested_is_proven"] is False + + def test_heap_keys_canonical_and_values_match(self, tmp_path: Path) -> None: + report = _build_real_direct_report() + report_path = _write_report(tmp_path, report) + summary = validate_and_aggregate_report(report_path) + keys = sorted(summary["heap_maxima_bytes"].keys()) + assert keys == ["js_heap_used_bytes_after", "js_heap_used_bytes_before"] + # counter = 1..120 => heap_before = 1000 + counter => max=1120 + assert summary["heap_maxima_bytes"]["js_heap_used_bytes_before"] == pytest.approx(1120.0) + assert summary["heap_maxima_bytes"]["js_heap_used_bytes_after"] == pytest.approx(2120.0) + + def test_completeness_passes_with_real_reports_120(self, tmp_path: Path) -> None: + report = _build_real_direct_report() + report_path = _write_report(tmp_path, report) + summary = validate_and_aggregate_report(report_path) + assert summary["measurement_completeness"]["complete"] is True + assert summary["measurement_completeness"]["incomplete_reasons"] == [] + assert summary["sample_counts"]["unique_session_starts"] == 6 diff --git a/tests/test_browser_parity_reference.py b/tests/test_browser_parity_reference.py new file mode 100644 index 0000000000000000000000000000000000000000..1458ba880be69c61c5222682178ba8691d94935f --- /dev/null +++ b/tests/test_browser_parity_reference.py @@ -0,0 +1,350 @@ +"""Verifier regressions with tiny tokenizers and mocked CPU outputs only.""" + +import copy +import hashlib +import importlib.util +import json +import math +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +SPEC = importlib.util.spec_from_file_location( + "browser_parity_reference", Path(__file__).parents[1] / "tools/verify_browser_parity.py" +) +assert SPEC is not None and SPEC.loader is not None +verifier = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(verifier) + + +class TinyTokenizer: + def encode(self, text): + ids = [101, *text.encode("utf-8"), 102] + return SimpleNamespace(ids=ids, attention_mask=[1] * len(ids), type_ids=[0] * len(ids)) + + def token_to_id(self, token): + assert token == "[PAD]" + return 0 + + +class Reference: + def __init__(self): + self.feeds = [] + self.answerability = 2.0 + self.invalid = False + + def run(self, feed, backend): + self.feeds.append(copy.deepcopy(feed)) + scores = np.zeros(feed["option_mask"].shape, np.float32) + scores[0, 0] = float("nan") if self.invalid else 3.0 + return scores, np.array([self.answerability], np.float32) + + +def question(key="q", kind="choice"): + return {"id": key, "type": kind, "prompt": "Choose", "options": ["first", "second"]} + + +def expected_answer(q, logit=2.0): + options = q["options"] or ["true", "false"] + probability = 1 / (1 + math.exp(-3)) + answerability = 1 / (1 + math.exp(-logit)) + confidence = probability * answerability + reason = "answerability_below_threshold" if answerability < 0.5 else ( + "confidence_below_threshold" if confidence < 0.55 else None + ) + return {"question_id": q["id"], "choice": None if reason else options[0], + "probabilities": [{"option": options[0], "probability": probability}, + {"option": options[1], "probability": 1 - probability}], + "confidence": confidence, "status": "abstain" if reason else "ok", + "abstain_reason": reason} + + +def fixture(backend="direct", multi=False): + questions = [question()] + if multi: + questions.append({"id": "q2", "type": "boolean", "prompt": "Continue?", "options": []}) + request = {"state": "ready", "questions": questions, "seed": 7} + metadata = {"backend": backend, "sequence_length": 128, "option_count": 4} + manifest = {"metadata": metadata} + noise = [0.25, -0.5, 1.0, 2.0] if backend == "diffusion" else None + traces, timings, answers = [], [], [] + for q in questions: + options = q["options"] or ["true", "false"] + slots = 4 if backend == "diffusion" else 2 + ids, masks, types, live = [], [], [], 0 + for index in range(slots): + raw = [101, *f"ready\nQuestion: {q['prompt']}\nCandidate: {options[index]}".encode(), 102] \ + if index < 2 else [] + live += len(raw) + ids.extend(raw + [0] * (128 - len(raw))) + masks.extend([1] * len(raw) + [0] * (128 - len(raw))) + types.extend([0] * 128) + answer = expected_answer(q) + traces.append({"questionId": q["id"], "inputIds": ids, "attentionMask": masks, + "tokenTypeIds": types, "optionMask": [1, 1] + [0] * (slots - 2), + "initialNoise": noise, "rawScores": [3.0, 0.0], "answerabilityLogit": 2.0, + "probabilities": [p["probability"] for p in answer["probabilities"]], + "answer": copy.deepcopy(answer)}) + timings.append({"questionId": q["id"], "live_candidates": 2, "allocated_candidates": slots, + "live_tokens": live, "sequence_length": 128}) + answers.append(answer) + row = {"fixture_id": "fixture", "input_hash": verifier.request_hash(request), + "request": request, "status": "ok", "traces": traces, "timings": timings, + "response": {"backend": backend, "model_id": f"vons-{backend}-onnx-web", + "answers": answers}} + report = {"schema": verifier.SCHEMA, "backend": backend, "fixture_count": 1, + "noise_values": noise, "records": [row]} + return report, manifest + + +def run(report, manifest, reference=None): + return verifier.verify_records(report, manifest, TinyTokenizer(), reference or Reference()) + + +def test_canonical_hash_matches_browser_fixture_domain_and_unicode_values(): + value = {"state": {"z": "🌐 검토", "a": 2}, "seed": 7, "questions": []} + canonical = '{"questions":[],"seed":7,"state":{"a":2,"z":"🌐 검토"}}' + assert verifier.stable_json(value) == canonical + assert verifier.request_hash(value) == hashlib.sha256(canonical.encode()).hexdigest() + assert verifier.stable_json({"seed": -0.0}) == '{"seed":0}' + + +@pytest.mark.parametrize("value", [{"Mixed": 1}, {"a_b": 1}, {"é": 1}, {"a": 1.25}, + {"a": 2**53}, {"a": float("nan")}]) +def test_unsupported_json_canonicalization_fails_closed(value): + with pytest.raises(verifier.ParityError): + verifier.stable_json(value) + + +@pytest.mark.parametrize("backend", ["direct", "diffusion"]) +def test_every_multiquestion_trace_and_response_compares(backend): + report, manifest = fixture(backend, multi=True) + reference = Reference() + result = run(report, manifest, reference) + assert result["counts"] == {"passed": 1, "failed": 0, "unverified": 0, + "contract_error_verified": 0} + assert result["numerical_questions"] == {"passed": 2, "failed": 0} + assert result["all_numerical_comparisons_passed"] + assert len(reference.feeds) == 2 + assert result["calibration"] is None + if backend == "diffusion": + expected = hashlib.sha256(np.array(report["noise_values"], dtype=" tuple[Path, Path]: + onnx = pytest.importorskip("onnx") + from onnx import TensorProto, helper, numpy_helper + + root.mkdir(parents=True, exist_ok=True) + initializer = numpy_helper.from_array(__import__("numpy").ones(2048, dtype="float32"), name="weight") + graph = helper.make_graph( + [helper.make_node("Identity", ["input"], ["output"])], + "test", + [helper.make_tensor_value_info("input", TensorProto.FLOAT, [2])], + [helper.make_tensor_value_info("output", TensorProto.FLOAT, [2])], + [initializer], + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 18)]) + graph_path = root / "model.onnx" + onnx.save_model(model, graph_path, save_as_external_data=True, all_tensors_to_one_file=True, location="weights.onnx.data") + tokenizer = root / "tokenizer" + tokenizer.mkdir() + (tokenizer / "tokenizer.json").write_text("{}\n", encoding="utf-8") + return graph_path, tokenizer + + +def test_manifest_inventories_external_data_and_verifies_bytes(tmp_path: Path) -> None: + graph, tokenizer = _write_external_model(tmp_path) + output = tmp_path / "bundle-manifest.v1.json" + build_bundle_manifest(graph, tokenizer, output) + result = verify_bundle_manifest(output) + payload = json.loads(output.read_text(encoding="utf-8")) + + assert result["pass"] is True + assert result["external_data_files"] == 1 + assert {item["role"] for item in payload["files"]} >= {"model_graph", "model_external_data", "tokenizer"} + assert payload["summary"]["model_asset_bytes"] == sum( + item["bytes"] for item in payload["files"] if item["model_asset"] + ) + + external = tmp_path / "weights.onnx.data" + external.write_bytes(external.read_bytes() + b"changed") + with pytest.raises(BundleManifestError, match="changed"): + verify_bundle_manifest(output) + + +def test_manifest_rejects_missing_external_data_and_empty_tokenizer(tmp_path: Path) -> None: + graph, tokenizer = _write_external_model(tmp_path) + (tmp_path / "weights.onnx.data").unlink() + with pytest.raises(BundleManifestError, match="external-data file does not exist"): + build_bundle_manifest(graph, tokenizer, tmp_path / "manifest.json") + + graph, tokenizer = _write_external_model(tmp_path / "second") + for child in tokenizer.iterdir(): + child.unlink() + with pytest.raises(BundleManifestError, match="contains no regular files"): + build_bundle_manifest(graph, tokenizer, tokenizer.parent / "manifest.json") + + +def test_manifest_rejects_external_data_traversal(tmp_path: Path) -> None: + onnx = pytest.importorskip("onnx") + from onnx import TensorProto, helper + + initializer = helper.make_tensor("weight", TensorProto.FLOAT, [1], [1.0]) + initializer.data_location = TensorProto.EXTERNAL + initializer.external_data.add(key="location", value="../outside.onnx.data") + graph = helper.make_graph([], "test", [], [], [initializer]) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 18)]) + graph_path = tmp_path / "model.onnx" + onnx.save_model(model, graph_path) + tokenizer = tmp_path / "tokenizer" + tokenizer.mkdir() + (tokenizer / "tokenizer.json").write_text("{}\n", encoding="utf-8") + + with pytest.raises(BundleManifestError, match="traversal"): + build_bundle_manifest(graph_path, tokenizer, tmp_path / "manifest.json") + + +def test_manifest_rejects_symlinked_tokenizer(tmp_path: Path) -> None: + graph, tokenizer = _write_external_model(tmp_path) + target = tmp_path / "outside.txt" + target.write_text("outside", encoding="utf-8") + (tokenizer / "escape.txt").symlink_to(target) + + with pytest.raises(BundleManifestError, match="symlink"): + build_bundle_manifest(graph, tokenizer, tmp_path / "manifest.json") + + +def test_source_snapshot_ignores_generated_python_caches(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "module.py").write_text("value = 1\n", encoding="utf-8") + ignored = source / "__pycache__" + ignored.mkdir() + (ignored / "module.cpython-313.pyc").write_bytes(b"first") + first = _snapshot_hash(source) + (ignored / "module.cpython-313.pyc").write_bytes(b"second") + (source / ".pytest_cache").mkdir() + (source / ".pytest_cache" / "state").write_bytes(b"cache") + assert _snapshot_hash(source) == first + + +def test_source_snapshot_can_pin_an_explicit_file_list(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "module.py").write_text("value = 1\n", encoding="utf-8") + (source / "config.json").write_text("{}\n", encoding="utf-8") + + first = _snapshot_hash(source, ["module.py"]) + assert first["scope"] == "explicit" + assert first["paths"] == ["module.py"] + (source / "config.json").write_text('{"changed": true}\n', encoding="utf-8") + assert _snapshot_hash(source, ["module.py"]) == first + (source / "module.py").write_text("value = 2\n", encoding="utf-8") + assert _snapshot_hash(source, ["module.py"]) != first + + +def test_source_snapshot_rejects_explicit_paths_outside_root(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + outside = tmp_path / "outside.py" + outside.write_text("value = 1\n", encoding="utf-8") + + with pytest.raises(BundleManifestError, match="escapes"): + _snapshot_hash(source, [outside]) diff --git a/tests/test_calibration.py b/tests/test_calibration.py new file mode 100644 index 0000000000000000000000000000000000000000..22e14bbd4a485a53b52423e68fcfe70fd38c8b98 --- /dev/null +++ b/tests/test_calibration.py @@ -0,0 +1,241 @@ +import hashlib +import json +import math +import tempfile +import unittest +from pathlib import Path + +from vons.calibration import ( + ABSTAIN_ALL_THRESHOLD, + CALIBRATION_VERSION, + apply_calibration, + calibrate_report, + fit_abstention_threshold, + fit_abstention_threshold_details, + fit_temperature, + fit_temperature_details, +) +from vons.data import Example, write_jsonl +from vons.evaluation import METRIC_VERSION, Prediction, nll, summarize, write_report + + +def _example(identifier: str, label: str | None, answerable: bool, split: str = "calibration") -> Example: + return Example( + id=identifier, + task_group="calibration", + state="state", + question="choose", + options=("a", "b"), + label=label, + answerable=answerable, + split=split, + provenance={}, + metadata={"scenario_id": identifier}, + ) + + +def _reject_constant(value: str) -> float: + raise ValueError(f"non-strict JSON constant {value}") + + +class CalibrationTests(unittest.TestCase): + def test_temperature_fit_finds_interior_optimum_and_does_not_increase_nll(self) -> None: + examples = [_example(f"r{index}", "a" if index < 3 else "b", True) for index in range(4)] + predictions = [Prediction(f"r{index}", "a", {"a": 0.6, "b": 0.4}, 0.6, False) for index in range(4)] + + details = fit_temperature_details(examples, predictions) + calibrated = apply_calibration(examples, predictions, temperature=details.temperature, abstention_threshold=0.0) + + self.assertAlmostEqual(details.temperature, math.log(1.5) / math.log(3.0), places=6) + self.assertEqual(details.rows, 4) + self.assertFalse(details.at_lower_bound) + self.assertFalse(details.at_upper_bound) + self.assertEqual(fit_temperature(examples, predictions), details.temperature) + self.assertLess(nll(examples, calibrated), nll(examples, predictions)) + + def test_temperature_bound_hit_is_flagged(self) -> None: + examples = [_example(f"s{index}", "a", True) for index in range(4)] + predictions = [Prediction(f"s{index}", "a", {"a": 0.9, "b": 0.1}, 0.9, False) for index in range(4)] + + details = fit_temperature_details(examples, predictions) + + self.assertTrue(details.at_lower_bound) + self.assertAlmostEqual(details.temperature, 0.05) + + def test_temperature_scaling_preserves_true_zero_support(self) -> None: + examples = [_example("zero", "a", True)] + predictions = [Prediction("zero", "a", {"a": 1.0, "b": 0.0}, 1.0, False)] + + calibrated = apply_calibration(examples, predictions, temperature=20.0, abstention_threshold=0.0) + + self.assertEqual(calibrated[0].probabilities, {"a": 1.0, "b": 0.0}) + + def test_invalid_probability_vectors_fail_closed(self) -> None: + examples = [_example("x", "a", True)] + for probabilities in ({"a": 0.5, "b": 0.2}, {"a": math.nan, "b": 1.0}, {"a": -0.5, "b": 1.5}): + with self.subTest(probabilities=probabilities), self.assertRaises(ValueError): + apply_calibration( + examples, + [Prediction("x", "a", probabilities, 0.5, False)], + temperature=1.0, + abstention_threshold=0.0, + ) + + def test_calibration_recomputes_argmax_and_never_covers_without_choice(self) -> None: + examples = [_example("was_abstained", "a", True), _example("stale_choice", "a", True)] + predictions = [ + Prediction("was_abstained", None, {"a": 0.6, "b": 0.4}, 0.42, True), + Prediction("stale_choice", "b", {"a": 0.6, "b": 0.4}, 0.6, False), + ] + + calibrated = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=0.2) + + self.assertEqual([(item.choice, item.abstained) for item in calibrated], [("a", False), ("a", False)]) + + def test_answerability_gate_is_preserved_and_labels_do_not_gate(self) -> None: + examples = [_example("low_answerability", "a", True), _example("unanswerable", None, False)] + predictions = [ + Prediction("low_answerability", "a", {"a": 0.6, "b": 0.4}, 0.2, False), + Prediction("unanswerable", "a", {"a": 0.6, "b": 0.4}, 0.6, False), + ] + + default = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=0.0) + relaxed = apply_calibration( + examples, + predictions, + temperature=1.0, + abstention_threshold=0.0, + answerability_threshold=0.3, + ) + + self.assertEqual((default[0].choice, default[0].abstained), (None, True)) + self.assertEqual((default[1].choice, default[1].abstained), ("a", False)) + self.assertEqual((relaxed[0].choice, relaxed[0].abstained), ("a", False)) + + def test_failed_and_absent_distributions_remain_abstained(self) -> None: + examples = [_example("failed", "a", True), _example("absent", "a", True)] + predictions = [ + Prediction("failed", None, {}, 0.0, False, error="timeout"), + Prediction("absent", "a", {}, 0.9, False), + ] + + calibrated = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=0.0) + + self.assertEqual((calibrated[0].choice, calibrated[0].abstained, calibrated[0].error), (None, True, "timeout")) + self.assertEqual((calibrated[1].choice, calibrated[1].abstained, calibrated[1].probabilities), (None, True, {})) + + def test_confidence_above_max_probability_fails_closed(self) -> None: + examples = [_example("x", "a", True)] + with self.assertRaisesRegex(ValueError, "max_probability"): + apply_calibration( + examples, + [Prediction("x", "a", {"a": 0.5, "b": 0.5}, 0.9, False)], + temperature=1.0, + abstention_threshold=0.0, + ) + + def test_threshold_fit_uses_argmax_and_counts_unanswerable_as_errors(self) -> None: + examples = [_example("correct", "a", True), _example("stale_choice", "a", True), _example("unknown", None, False)] + predictions = [ + Prediction("correct", "a", {"a": 0.9, "b": 0.1}, 0.9, False), + Prediction("stale_choice", "b", {"a": 0.6, "b": 0.4}, 0.58, False), + Prediction("unknown", "a", {"a": 0.55, "b": 0.45}, 0.5, False), + ] + + details = fit_abstention_threshold_details(examples, predictions, target_risk=0.0) + calibrated = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=details.threshold) + + self.assertAlmostEqual(details.threshold, 0.58) + self.assertEqual((details.rows, details.covered_rows, details.risk, details.abstain_all), (3, 2, 0.0, False)) + self.assertEqual(fit_abstention_threshold(examples, predictions, target_risk=0.0), details.threshold) + self.assertEqual([item.abstained for item in calibrated], [False, False, True]) + + def test_all_abstain_sentinel_covers_nothing_even_at_confidence_one(self) -> None: + examples = [_example("wrong", "a", True)] + predictions = [Prediction("wrong", "b", {"a": 0.0, "b": 1.0}, 1.0, False)] + + threshold = fit_abstention_threshold(examples, predictions, target_risk=0.0) + at_one = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=1.0)[0] + sentinel = apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=threshold)[0] + + self.assertEqual(threshold, ABSTAIN_ALL_THRESHOLD) + self.assertFalse(at_one.abstained) + self.assertEqual((sentinel.choice, sentinel.abstained), (None, True)) + for invalid in (1.5, -0.1, math.nan): + with self.subTest(threshold=invalid), self.assertRaisesRegex(ValueError, "abstention_threshold"): + apply_calibration(examples, predictions, temperature=1.0, abstention_threshold=invalid) + + def test_unknown_or_duplicate_prediction_ids_fail_closed(self) -> None: + examples = [_example("a", "a", True)] + with self.assertRaisesRegex(ValueError, "unknown example id"): + apply_calibration( + examples, + [Prediction("ghost", "a", {"a": 1.0}, 1.0, False)], + temperature=1.0, + abstention_threshold=0.0, + ) + with self.assertRaisesRegex(ValueError, "duplicate prediction"): + fit_abstention_threshold(examples, [Prediction("a", "a", {"a": 1.0}, 1.0, False)] * 2) + + def test_calibrate_report_records_provenance_and_rejects_identity_mismatch(self) -> None: + config = { + "model_id": "vons-test", + "backend": "direct", + "seed": 7, + "encoder": {"name": "encoder", "revision": "r1"}, + "answerability_threshold": 0.8, + } + calibration_rows = [_example("c1", "a", True), _example("c2", "b", True), _example("c3", None, False)] + calibration_predictions = [ + Prediction("c1", "a", {"a": 0.9, "b": 0.1}, 0.9, False), + Prediction("c2", "b", {"a": 0.3, "b": 0.7}, 0.63, False), + Prediction("c3", None, {"a": 0.6, "b": 0.4}, 0.3, True), + ] + test_rows = [_example("t1", "a", True, "test"), _example("t2", "b", True, "test"), _example("t3", None, False, "test")] + test_predictions = [ + Prediction("t1", "a", {"a": 0.8, "b": 0.2}, 0.8, False), + Prediction("t2", "b", {"a": 0.4, "b": 0.6}, 0.42, False), + Prediction("t3", None, {}, 0.0, False, error="timeout"), + ] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + calibration_input, calibration_report = root / "calibration.jsonl", root / "calibration.json" + test_input, test_report, output = root / "test.jsonl", root / "test.json", root / "calibrated.json" + write_jsonl(calibration_input, calibration_rows) + write_jsonl(test_input, test_rows) + write_report(calibration_report, config=config, summary=summarize(calibration_rows, calibration_predictions), predictions=calibration_predictions) + write_report(test_report, config=config, summary=summarize(test_rows, test_predictions), predictions=test_predictions) + arguments = { + "calibration_input": calibration_input, + "calibration_report": calibration_report, + "test_input": test_input, + "test_report": test_report, + } + + calibrate_report(**arguments, output=output, target_risk=0.0) + payload = json.loads(output.read_text(encoding="utf-8"), parse_constant=_reject_constant) + + fit = payload["calibration"] + self.assertEqual(fit["version"], CALIBRATION_VERSION) + self.assertEqual(fit["answerability_threshold"], 0.8) + self.assertEqual((fit["temperature_fit_rows"], fit["threshold_fit_rows"], fit["rows"]), (2, 3, 3)) + self.assertTrue(fit["temperature_at_lower_bound"]) + self.assertLessEqual(fit["nll_after"], fit["nll_before"]) + self.assertEqual(payload["inputs"]["test_report"]["sha256"], hashlib.sha256(test_report.read_bytes()).hexdigest()) + self.assertEqual(payload["model_identity"]["model_id"], "vons-test") + self.assertTrue(any("checkpoint_sha256 unavailable" in item for item in payload["limitations"])) + self.assertEqual(payload["summary"]["metric_version"], METRIC_VERSION) + decisions = {item["example_id"]: (item["choice"], item["abstained"]) for item in payload["predictions"]} + self.assertEqual(decisions, {"t1": ("a", False), "t2": (None, True), "t3": (None, True)}) + + mismatched = root / "mismatched.json" + write_report(mismatched, config={**config, "model_id": "other"}, summary={}, predictions=test_predictions) + with self.assertRaisesRegex(ValueError, "model_id"): + calibrate_report(**{**arguments, "test_report": mismatched}, output=root / "rejected.json") + with self.assertRaisesRegex(ValueError, "overwrite"): + calibrate_report(**arguments, output=test_report) + self.assertFalse((root / "rejected.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_contract.py b/tests/test_contract.py new file mode 100644 index 0000000000000000000000000000000000000000..97cfd4f522af6599ca923f49ff02586082bbc1fd --- /dev/null +++ b/tests/test_contract.py @@ -0,0 +1,40 @@ +import unittest + +from vons.contract import ( + Backend, + DecisionRequest, + Question, + QuestionType, + derive_kasi_action, + validate_request, +) +from vons.kasi import KASIAdapter, KASIProposal + + +class ContractTests(unittest.TestCase): + def test_valid_request(self) -> None: + validate_request(DecisionRequest("A weather tool is available.", (Question("next", QuestionType.CHOICE, "Choose.", ("call", "clarify")),), Backend.DIRECT, 7)) + + def test_duplicate_questions_rejected(self) -> None: + request = DecisionRequest("state", (Question("q", QuestionType.CHOICE, "pick", ("a", "b")), Question("q", QuestionType.CHOICE, "pick", ("a", "b")))) + with self.assertRaises(ValueError): + validate_request(request) + + def test_kasi_confirm_is_wrapper_derived(self) -> None: + self.assertEqual(derive_kasi_action(proposed_calls=({"name": "unlock"},), confidence=0.99, risk="high", confirmed_tools=set(), tool_policy={"unlock": "high"}).value, "confirm") + self.assertEqual(derive_kasi_action(proposed_calls=({"name": "read"},), confidence=0.99, risk="low", confirmed_tools=set(), tool_policy={"read": "low"}).value, "call") + + def test_unregistered_tool_cannot_call(self) -> None: + action = derive_kasi_action(proposed_calls=({"name": "read"},), confidence=0.99, risk="low", tool_policy={}) + self.assertEqual(action.value, "refuse") + + def test_kasi_response_and_low_confidence(self) -> None: + adapter = KASIAdapter(tool_policy={"read_weather": "low"}) + response = adapter.decide(KASIProposal(response="The answer is ready.", confidence=0.9)) + self.assertEqual(response.action.value, "respond") + clarify = adapter.decide(KASIProposal(calls=(), confidence=0.2)) + self.assertEqual(clarify.action.value, "clarify") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_data.py b/tests/test_data.py new file mode 100644 index 0000000000000000000000000000000000000000..415c4a2f4db9802d968b139be923e5a9c3357fe3 --- /dev/null +++ b/tests/test_data.py @@ -0,0 +1,23 @@ +import tempfile +import unittest +from pathlib import Path + +from vons.data import dataset_manifest, read_jsonl, smoke_examples, synthetic_examples, write_jsonl + + +class DataTests(unittest.TestCase): + def test_round_trip(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "smoke.jsonl" + write_jsonl(path, smoke_examples()) + rows = read_jsonl(path) + self.assertEqual(len(rows), 4) + self.assertEqual(dataset_manifest(path, rows)["rows"], 4) + + def test_reproducible_generation(self) -> None: + self.assertEqual([row.to_mapping() for row in synthetic_examples(20, seed=9)], [row.to_mapping() for row in synthetic_examples(20, seed=9)]) + + +if __name__ == "__main__": + unittest.main() + diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..19fb08b160eb203deeb7a185b3972ee96bc3b5c5 --- /dev/null +++ b/tests/test_evaluation.py @@ -0,0 +1,208 @@ +import json +import math +import tempfile +import unittest +from pathlib import Path + +from vons.data import Example, smoke_examples +from vons.evaluation import ( + METRIC_VERSION, + Prediction, + accuracy, + brier_score, + coverage, + ece, + macro_f1, + nll, + selective_risk, + summarize, + validate_distribution, + write_report, +) + + +def _row(identifier: str, label: str | None, *, answerable: bool = True, options: tuple[str, ...] = ("a", "b")) -> Example: + return Example( + id=identifier, + task_group="evaluation", + state="state", + question="choose", + options=options, + label=label, + answerable=answerable, + split="test", + provenance={}, + metadata={}, + ) + + +def _reject_constant(value: str) -> float: + raise ValueError(f"non-strict JSON constant {value}") + + +class EvaluationTests(unittest.TestCase): + def test_metrics_include_abstention_and_probability_checks(self) -> None: + rows = smoke_examples()[:3] + predictions = [ + Prediction(row.id, row.label, {option: 1.0 if option == row.label else 0.0 for option in row.options}, 1.0, False) + for row in rows + ] + self.assertEqual(macro_f1(rows, predictions), 1.0) + self.assertEqual(nll(rows, predictions), 0.0) + self.assertEqual(brier_score(rows, predictions), 0.0) + self.assertEqual(selective_risk(rows, predictions), 0.0) + summary = summarize(rows, predictions) + self.assertIn("macro_f1_answerable", summary) + self.assertEqual(summary["metric_version"], METRIC_VERSION) + self.assertIn("brier_answerable", summary["metric_definitions"]) + self.assertEqual(summary["probability_population"], "complete") + + def test_missing_predictions_stay_in_accuracy_and_coverage_denominators(self) -> None: + rows = [_row(f"r{index}", "a") for index in range(10)] + predictions = [Prediction("r0", "a", {"a": 0.99, "b": 0.01}, 0.99, False)] + + summary = summarize(rows, predictions) + + self.assertEqual(summary["accuracy_answerable"], 0.1) + self.assertEqual(summary["coverage"], 0.1) + self.assertEqual(summary["selective_risk"], 0.0) + self.assertEqual(summary["counts"]["missing_prediction_rows"], 9) + self.assertEqual(summary["counts"]["probability_rows"], 1) + self.assertEqual(summary["counts"]["probability_unavailable_answerable_rows"], 9) + self.assertEqual(summary["probability_population"], "incomplete") + self.assertAlmostEqual(summary["nll_answerable"], -math.log(0.99)) + + def test_abstained_and_error_predictions_are_never_correct_or_covered(self) -> None: + rows = [_row("abstained", "a"), _row("failed", "a")] + predictions = [ + Prediction("abstained", "a", {"a": 0.9, "b": 0.1}, 0.9, True), + Prediction("failed", "a", {}, 0.0, False, error="timeout"), + ] + + summary = summarize(rows, predictions) + + self.assertEqual(accuracy(rows, predictions), 0.0) + self.assertEqual(macro_f1(rows, predictions), 0.0) + self.assertEqual(coverage(rows, predictions), 0.0) + self.assertIsNone(selective_risk(rows, predictions)) + self.assertEqual(summary["counts"]["abstained_prediction_rows"], 1) + self.assertEqual(summary["counts"]["error_prediction_rows"], 1) + self.assertEqual(summary["counts"]["covered_rows"], 0) + self.assertEqual(summary["counts"]["probability_rows"], 1) + + def test_covered_unanswerable_prediction_is_a_selective_risk_error(self) -> None: + rows = [_row("answerable", "a"), _row("unanswerable", None, answerable=False)] + predictions = [ + Prediction("answerable", "a", {"a": 0.9, "b": 0.1}, 0.9, False), + Prediction("unanswerable", "a", {"a": 0.9, "b": 0.1}, 0.9, False), + ] + + self.assertEqual(selective_risk(rows, predictions), 0.5) + self.assertEqual(coverage(rows, predictions), 1.0) + self.assertAlmostEqual(ece(rows, predictions), 0.4) + self.assertEqual(summarize(rows, predictions)["counts"]["covered_unanswerable_rows"], 1) + + def test_undefined_metrics_are_none(self) -> None: + rows = [_row("a", "a"), _row("b", "b")] + abstained = [ + Prediction("a", None, {"a": 0.2, "b": 0.8}, 0.2, True), + Prediction("b", None, {"a": 0.9, "b": 0.1}, 0.1, True), + ] + self.assertIsNone(selective_risk(rows, abstained)) + self.assertIsNone(ece(rows, abstained)) + self.assertEqual(coverage(rows, abstained), 0.0) + + unanswerable = [_row("u", None, answerable=False)] + answered = [Prediction("u", "a", {"a": 0.5, "b": 0.5}, 0.5, False)] + for metric in (accuracy, macro_f1, nll, brier_score): + self.assertIsNone(metric(unanswerable, answered)) + self.assertIsNone(coverage([], [])) + + def test_macro_f1_ignores_choices_outside_answerable_rows(self) -> None: + rows = [_row("a", "a", options=("a", "b", "zzz")), _row("b", "b", options=("a", "b", "zzz")), _row("u", None, answerable=False, options=("a", "b", "zzz"))] + predictions = [ + Prediction("a", "a", {}, 0.9, False), + Prediction("b", "b", {}, 0.9, False), + Prediction("u", "zzz", {}, 0.9, False), + ] + self.assertEqual(macro_f1(rows, predictions), 1.0) + + def test_unknown_and_duplicate_ids_fail_closed(self) -> None: + rows = [_row("a", "a")] + with self.assertRaisesRegex(ValueError, "unknown example id"): + summarize(rows, [Prediction("ghost", "a", {}, 0.9, False)]) + with self.assertRaisesRegex(ValueError, "duplicate prediction"): + summarize(rows, [Prediction("a", "a", {}, 0.9, False), Prediction("a", "b", {}, 0.9, False)]) + with self.assertRaisesRegex(ValueError, "duplicate example id"): + summarize([_row("a", "a"), _row("a", "a")], []) + + def test_invalid_confidence_and_probability_vectors_fail_closed(self) -> None: + rows = [_row("a", "a")] + for confidence in (math.nan, -0.1, 1.8): + with self.subTest(confidence=confidence), self.assertRaisesRegex(ValueError, "confidence"): + ece(rows, [Prediction("a", "a", {"a": 1.0}, confidence, False)]) + for probabilities in ({"a": math.nan, "b": 0.5}, {"a": -0.1, "b": 1.1}, {"a": 0.5, "b": 0.2}, {"a": 0.5, "c": 0.5}): + with self.subTest(probabilities=probabilities), self.assertRaises(ValueError): + nll(rows, [Prediction("a", "a", probabilities, 0.5, False)]) + with self.assertRaisesRegex(ValueError, "no choice"): + coverage(rows, [Prediction("a", None, {"a": 1.0}, 1.0, False)]) + + def test_answerable_row_without_label_fails_closed(self) -> None: + rows = [_row("a", None)] + with self.assertRaisesRegex(ValueError, "needs a label"): + accuracy(rows, [Prediction("a", None, {}, 0.1, True)]) + + def test_brier_score_is_the_standard_multiclass_sum(self) -> None: + for options, expected in ((("a", "b"), 0.5), (("a", "b", "c", "d"), 0.75)): + rows = [_row("r", "a", options=options)] + uniform = {option: 1.0 / len(options) for option in options} + self.assertAlmostEqual(brier_score(rows, [Prediction("r", "a", uniform, 0.5, False)]), expected) + + def test_probability_metrics_are_conditional_on_valid_distributions(self) -> None: + rows = [_row("valid", "a"), _row("failed", "a"), _row("absent", "a"), _row("zero", "a")] + predictions = [ + Prediction("valid", "a", {"a": 0.8, "b": 0.2}, 0.8, False), + Prediction("failed", None, {}, 0.0, False, error="crash"), + Prediction("absent", "a", {}, 0.7, False), + Prediction("zero", "b", {"b": 1.0}, 1.0, False), + ] + + summary = summarize(rows, predictions) + + self.assertEqual(summary["counts"]["probability_rows"], 2) + self.assertEqual(summary["counts"]["probability_unavailable_answerable_rows"], 2) + self.assertEqual(summary["counts"]["nll_floor_clipped_rows"], 1) + self.assertEqual(summary["probability_population"], "incomplete") + self.assertAlmostEqual(summary["nll_answerable"], (-math.log(0.8) - math.log(1e-12)) / 2) + self.assertEqual(validate_distribution({"b": 1.0}, ("a", "b")), {"b": 1.0}) + + def test_ece_uses_last_bin_for_confidence_one(self) -> None: + rows = [_row("a", "a"), _row("b", "b"), _row("c", "a"), _row("u", None, answerable=False)] + predictions = [ + Prediction("a", "a", {"a": 0.9, "b": 0.1}, 0.9, False), + Prediction("b", "a", {"a": 0.9, "b": 0.1}, 0.9, False), + Prediction("c", "a", {"a": 1.0}, 1.0, False), + Prediction("u", "b", {"a": 0.3, "b": 0.7}, 0.3, False), + ] + self.assertAlmostEqual(ece(rows, predictions), 0.275) + with self.assertRaisesRegex(ValueError, "bins"): + ece(rows, predictions, bins=0) + + def test_write_report_is_strict_json(self) -> None: + rows = [_row("a", "a")] + predictions = [Prediction("a", "a", {"a": 1.0}, 1.0, False, latency_ms=1.5)] + with tempfile.TemporaryDirectory() as directory: + invalid = Path(directory) / "invalid.json" + with self.assertRaises(ValueError): + write_report(invalid, config={}, summary={"nll_answerable": math.nan}, predictions=predictions) + self.assertFalse(invalid.exists()) + + valid = Path(directory) / "valid.json" + write_report(valid, config={"model_id": "m"}, summary=summarize(rows, predictions), predictions=predictions) + payload = json.loads(valid.read_text(encoding="utf-8"), parse_constant=_reject_constant) + self.assertEqual(payload["summary"]["metric_version"], METRIC_VERSION) + self.assertEqual(payload["summary"]["latency_p50_ms"], 1.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_merge_browser_benchmarks.py b/tests/test_merge_browser_benchmarks.py new file mode 100644 index 0000000000000000000000000000000000000000..06aa1f15e632da7f136826d444c4120c56faac66 --- /dev/null +++ b/tests/test_merge_browser_benchmarks.py @@ -0,0 +1,102 @@ +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + + +def _load_tool(name: str, file_name: str): + path = Path(__file__).parents[1] / "tools" / file_name + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not load {file_name}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +benchmark_tool = _load_tool("verify_browser_benchmark", "verify_browser_benchmark.py") +merge_tool = _load_tool("vons_merge_browser_benchmarks", "merge_browser_benchmarks.py") + + +def _sample(repeat: int) -> dict[str, object]: + return { + "questionId": f"case-{repeat + 1}", + "live_candidates": 4, + "allocated_candidates": 4, + "live_tokens": 12, + "sequence_length": 512, + "tokenizeMs": 1.0, + "prepareFeedMs": 2.0, + "inferenceAndReadbackMs": 3.0 + repeat, + "postprocessMs": 4.0, + "totalRequestMs": 10.0 + repeat, + "run_id": f"run-{repeat}", + "session_start": 1, + "repeat": repeat, + "case_id": f"case-{repeat + 1}", + "backend": "direct", + "requested_provider": "wasm", + "observed_provider": None, + "observed_provider_status": "session_created; execution_partition_not_exposed", + "bundle_hash": "c" * 64, + "tokenizer_hash": "a" * 64, + "input_hash": "b" * 64, + "noise_hash": None, + "js_heap_used_bytes_before": None, + "js_heap_used_bytes_after": None, + "status": "ok", + } + + +def _report(run_id: str, offset: int) -> dict[str, object]: + return { + "schema": "vons.browser-benchmark/v1", + "run_id": run_id, + "backend": "direct", + "requested_provider": "wasm", + "sessions": 1, + "warmup_excluded_per_session": 1, + "repeats_per_session": 2, + "cases": 2, + "session_start_semantics": "new ORT session on one page; module and browser caches may persist", + "execution_provider_evidence": "requested provider and session creation only; graph partition status is not exposed by this harness", + "environment": {"user_agent": "test", "cross_origin_isolated": False}, + "runtime_info": None, + "session_records": [{"session_start": 1, "load_ms": 5.0, "status": "ok"}], + "samples": [_sample(offset), _sample(offset + 1)], + } + + +class MergeBrowserBenchmarkTests(unittest.TestCase): + def test_merges_complete_one_session_reports(self) -> None: + with tempfile.TemporaryDirectory() as directory: + first = Path(directory) / "first.json" + second = Path(directory) / "second.json" + first.write_text(json.dumps(_report("first", 0)), encoding="utf-8") + second.write_text(json.dumps(_report("second", 0)), encoding="utf-8") + merged = merge_tool.merge_reports([first, second]) + + self.assertEqual(merged["sessions"], 2) + self.assertEqual(len(merged["samples"]), 4) + self.assertEqual({sample["session_start"] for sample in merged["samples"]}, {1, 2}) + self.assertEqual(merged["merge"]["source_count"], 2) + self.assertEqual(merged["summary"]["successful_samples"], 4) + + def test_rejects_mismatched_provider(self) -> None: + first_report = _report("first", 0) + second_report = _report("second", 0) + second_report["requested_provider"] = "webgpu" + with tempfile.TemporaryDirectory() as directory: + first = Path(directory) / "first.json" + second = Path(directory) / "second.json" + first.write_text(json.dumps(first_report), encoding="utf-8") + second.write_text(json.dumps(second_report), encoding="utf-8") + with self.assertRaises(ValueError): + merge_tool.merge_reports([first, second]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mind2web.py b/tests/test_mind2web.py new file mode 100644 index 0000000000000000000000000000000000000000..4567a47ff7b7fd9b49409f04ae5af4c2a19f289e --- /dev/null +++ b/tests/test_mind2web.py @@ -0,0 +1,142 @@ +import unittest + +from vons.mind2web import Mind2WebExample, evaluate_mind2web, recall_at_k, task_bootstrap_intervals + + +class Mind2WebEvaluationTests(unittest.TestCase): + def test_recall_and_selection_are_separate(self) -> None: + rows = [ + Mind2WebExample("a", ("x", "target"), "target"), + Mind2WebExample("b", ("target", "y"), "target"), + ] + metrics = evaluate_mind2web( + rows, + {"a": ("x", "target"), "b": ("y",)}, + {"a": "x", "b": "target"}, + ) + self.assertEqual(metrics.candidate_recall, 0.5) + self.assertEqual(metrics.selection_accuracy_given_recall, 0.0) + self.assertEqual(metrics.complete_case_selection_accuracy_given_recall, 0.0) + self.assertEqual(metrics.evaluated_recalled_rows, 1) + self.assertNotIn("browser_task_success", metrics.to_mapping()) + + def test_mapping_keeps_positive_ids_out_of_retrieved_candidates(self) -> None: + row = Mind2WebExample.from_mapping( + {"id": "a", "candidate_ids": ["x"], "target_id": "target"} + ) + self.assertEqual(row.candidate_ids, ("x",)) + self.assertEqual(row.positive_ids, ("target",)) + + retrieved = Mind2WebExample.from_mapping( + {"id": "a", "candidate_ids": ["target"], "target_id": "target"} + ) + self.assertEqual(retrieved.candidate_ids, ("target",)) + + def test_mapping_supports_multiple_positives_and_metadata(self) -> None: + row = Mind2WebExample.from_mapping( + { + "id": "a", + "candidate_ids": ["x", "y"], + "positive_ids": ["target", "target-alias"], + "task_id": "task-1", + "action_id": "action-2", + "split": "cross-website", + "website": "example.test", + "domain": "retail", + } + ) + self.assertEqual(row.positive_ids, ("target", "target-alias")) + self.assertEqual(row.target_id, "target") + self.assertEqual(row.split, "cross-website") + + def test_mapping_supports_explicit_no_positive_and_empty_candidates(self) -> None: + row = Mind2WebExample.from_mapping({"id": "none", "candidates": [], "no_positive": True}) + self.assertTrue(row.no_positive) + self.assertEqual(row.positive_ids, ()) + + def test_mapping_rejects_string_false_and_null_candidate_ids(self) -> None: + with self.assertRaisesRegex(TypeError, "no_positive"): + Mind2WebExample.from_mapping({"id": "none", "candidates": [], "no_positive": "false"}) + with self.assertRaisesRegex(TypeError, "ids"): + Mind2WebExample.from_mapping({"id": "bad", "candidates": [{"id": None}], "positive_ids": ["target"]}) + + def test_missing_prediction_is_excluded_from_conditional_accuracy(self) -> None: + rows = [ + Mind2WebExample.from_mapping( + {"id": "a", "candidate_ids": ["x"], "positive_ids": ["target"]} + ), + Mind2WebExample.from_mapping( + {"id": "b", "candidate_ids": ["x"], "positive_ids": ["target"]} + ), + ] + metrics = evaluate_mind2web( + rows, + {"a": ("target",), "b": ("target",)}, + {"a": None, "b": "target"}, + ) + self.assertEqual(metrics.candidate_recall, 1.0) + self.assertEqual(metrics.selection_accuracy_given_recall, 0.5) + self.assertEqual(metrics.complete_case_selection_accuracy_given_recall, 1.0) + self.assertEqual(metrics.missing_predictions, 1) + self.assertEqual(metrics.evaluated_recalled_rows, 1) + + def test_invalid_selection_is_evaluated_but_not_correct(self) -> None: + row = Mind2WebExample.from_mapping( + {"id": "a", "candidate_ids": ["x"], "positive_ids": ["target"]} + ) + metrics = evaluate_mind2web({row}, {"a": ("target", "x")}, {"a": "not-retrieved"}) + self.assertEqual(metrics.selection_accuracy_given_recall, 0.0) + self.assertEqual(metrics.invalid_selections, 1) + + def test_undefined_metrics_are_null_and_no_positive_is_separate(self) -> None: + row = Mind2WebExample.from_mapping({"id": "none", "no_positive": True}) + metrics = evaluate_mind2web([row], {"none": ()}, {"none": None}) + self.assertIsNone(metrics.candidate_recall) + self.assertIsNone(metrics.selection_accuracy_given_recall) + self.assertEqual(metrics.no_positive_rows, 1) + self.assertIsNone(metrics.to_mapping()["candidate_recall"]) + + def test_recall_at_k_uses_retriever_order(self) -> None: + row = Mind2WebExample.from_mapping( + {"id": "a", "candidate_ids": [], "positive_ids": ["target"]} + ) + self.assertEqual(recall_at_k([row], {"a": ("x", "target")}, 1), 0.0) + self.assertEqual(recall_at_k([row], {"a": ("x", "target")}, 2), 1.0) + + def test_task_macro_and_step_micro_metrics_are_separate(self) -> None: + rows = [ + Mind2WebExample("a1", ("target",), "target", task_id="task-a"), + Mind2WebExample("a2", ("x",), "target", task_id="task-a"), + Mind2WebExample("b1", ("target",), "target", task_id="task-b"), + ] + metrics = evaluate_mind2web( + rows, + {"a1": ("target",), "a2": ("x",), "b1": ("target",)}, + {"a1": "target", "b1": "target"}, + ) + self.assertEqual(metrics.candidate_recall, 2 / 3) + self.assertEqual(metrics.candidate_recall_task_macro, 0.75) + self.assertEqual(metrics.selection_accuracy_given_recall_task_macro, 1.0) + self.assertEqual(metrics.to_mapping()["candidate_recall_step_micro"], 2 / 3) + self.assertEqual(metrics.task_group_count, 2) + + def test_task_bootstrap_is_deterministic_and_task_weighted(self) -> None: + rows = [ + Mind2WebExample("a1", ("target",), "target", task_id="task-a"), + Mind2WebExample("a2", ("x",), "target", task_id="task-a"), + Mind2WebExample("b1", ("target",), "target", task_id="task-b"), + ] + args = ( + rows, + {"a1": ("target",), "a2": ("x",), "b1": ("target",)}, + {"a1": "target", "b1": "target"}, + ) + first = task_bootstrap_intervals(*args, draws=20, seed=11) + second = task_bootstrap_intervals(*args, draws=20, seed=11) + self.assertEqual(first, second) + self.assertEqual(first["task_group_count"], 2) + self.assertEqual(len(first["candidate_recall_task_macro_ci95"]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mind2web_import.py b/tests/test_mind2web_import.py new file mode 100644 index 0000000000000000000000000000000000000000..7642276be18363758abdd2eb296b1aad0f3d6ca6 --- /dev/null +++ b/tests/test_mind2web_import.py @@ -0,0 +1,368 @@ +"""Handcrafted fixtures only; official restricted tasks never live in tests.""" + +import copy +import hashlib +import importlib.util +import io +import json +import stat +import unittest +import zipfile +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from vons.mind2web import Mind2WebExample + + +def _load_importer(): + path = Path(__file__).parents[1] / "tools" / "import_mind2web.py" + spec = importlib.util.spec_from_file_location("vons_mind2web_importer", path) + if spec is None or spec.loader is None: + raise RuntimeError("could not load Mind2Web importer") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +importer = _load_importer() + + +def _candidate(node_id): + return { + "backend_node_id": node_id, + "tag": "button", + "is_original_target": True, + "is_top_level_target": True, + "attributes": json.dumps({"is_original_target": "SECRET_GOLD_FLAG", "role": "button"}), + } + + +def _task(task_id="task-a", action_id="action-a"): + return { + "annotation_id": task_id, + "website": "fixture.example", + "domain": "fixture-domain", + "confirmed_task": "Find the blue train", + "action_reprs": ["CURRENT_ACTION_SECRET", "FUTURE_ACTION_SECRET"], + "actions": [ + { + "action_uid": action_id, + "raw_html": "RAW_HTML_SECRET", + "cleaned_html": ( + '' + ), + "operation": {"op": "TYPE", "value": "OPERATION_VALUE_SECRET"}, + "pos_candidates": [_candidate("30"), _candidate("20")], + "neg_candidates": [_candidate("10")], + }, + { + "action_uid": action_id + "-next", + "cleaned_html": '', + "operation": {"op": "CLICK", "value": "NEXT_OPERATION_SECRET"}, + "pos_candidates": [], + "neg_candidates": [_candidate("40")], + }, + ], + } + + +def _manifest(): + return { + "schema": "vons.mind2web-acquisition/v1", + "dataset": "osunlp/Mind2Web", + "dataset_revision": importer.DATASET_REVISION, + "usage": "evaluation-only", + "restrictions": {"no_training": True, "no_unzipped_redistribution": True}, + "archive": { + "sha256": importer.ARCHIVE_SHA256, + "bytes": importer.ARCHIVE_BYTES, + "status": "downloaded-and-hash-verified", + }, + "extraction": {"status": "complete"}, + "files": [], + } + + +class Mind2WebImportTests(unittest.TestCase): + def test_multiple_positives_and_no_positive_preserve_contract(self): + rows = list(importer.normalize_task(_task(), "test_task")) + first = Mind2WebExample.from_mapping(rows[0][0]) + second = Mind2WebExample.from_mapping(rows[1][0]) + self.assertEqual(first.candidate_ids, ("10", "20", "30")) + self.assertEqual(first.positive_ids, ("20", "30")) + self.assertEqual(first.task_id, "task-a") + self.assertEqual(first.action_id, "action-a") + self.assertEqual(first.website, "fixture.example") + self.assertEqual(first.domain, "fixture-domain") + self.assertEqual(first.split, "test_task") + self.assertTrue(second.no_positive) + self.assertEqual(second.positive_ids, ()) + + def test_current_future_operation_raw_html_and_flags_never_enter_retriever(self): + rows = list(importer.normalize_task(_task(), "test_task")) + first = json.dumps(rows[0][1]) + for secret in ( + "CURRENT_ACTION_SECRET", + "FUTURE_ACTION_SECRET", + "OPERATION_VALUE_SECRET", + "RAW_HTML_SECRET", + "SECRET_GOLD_FLAG", + "DOM_GOLD_SECRET", + "positive_ids", + "no_positive", + "is_original_target", + ): + self.assertNotIn(secret, first) + self.assertEqual(rows[1][1]["state"]["previous_actions"], ["CURRENT_ACTION_SECRET"]) + self.assertNotIn("FUTURE_ACTION_SECRET", json.dumps(rows[1][1])) + + def test_relabeling_and_reordering_do_not_change_rank_or_input_hash(self): + task = _task() + original = next(importer.normalize_task(task, "test_task"))[1] + changed = copy.deepcopy(task) + action = changed["actions"][0] + action["neg_candidates"], action["pos_candidates"] = ( + list(reversed(action["pos_candidates"])), + action["neg_candidates"], + ) + action["operation"] = {"op": "SELECT", "value": "OTHER_SECRET"} + changed["action_reprs"] = ["DIFFERENT_CURRENT", "DIFFERENT_FUTURE"] + for field in ("pos_candidates", "neg_candidates"): + for candidate in action[field]: + candidate["is_original_target"] = False + candidate["is_top_level_target"] = False + result = next(importer.normalize_task(changed, "test_task"))[1] + self.assertEqual(original, result) + + def test_lexical_baseline_uses_dom_text_and_deterministic_ties(self): + rows = list(importer.normalize_task(_task(), "test_task")) + ranked = rows[0][1]["candidates"] + self.assertIn("Blue train", ranked[0]["text"]) + self.assertEqual([item["rank"] for item in ranked], [1, 2, 3]) + self.assertEqual(ranked[-1]["id"], "10") + candidates = [{"id": "z", "text": "same"}, {"id": "a", "text": "same"}] + self.assertEqual( + [row["id"] for row in importer.lexical_rank("unmatched", candidates)], ["a", "z"] + ) + + def test_xml_dom_preserves_candidates_inside_nested_iframes(self): + task = _task() + action = task["actions"][0] + action["cleaned_html"] = ( + '' + 'Blue timetable' + ) + candidates = importer.candidate_universe(action) + self.assertEqual([row["id"] for row in candidates], ["10", "20", "30"]) + self.assertIn("Alpha&Beta", candidates[0]["text"]) + self.assertIn("Blue train", candidates[1]["text"]) + + def test_empty_candidate_pool_is_preserved(self): + task = _task() + task["actions"][0]["pos_candidates"] = [] + task["actions"][0]["neg_candidates"] = [] + labels, retrieval = next(importer.normalize_task(task, "test_task")) + self.assertTrue(labels["no_positive"]) + self.assertEqual(retrieval["generated_candidates"], []) + + def test_duplicate_candidate_and_missing_cleaned_node_fail_closed(self): + task = _task() + task["actions"][0]["neg_candidates"].append(_candidate("30")) + with self.assertRaisesRegex(ValueError, "duplicate candidate"): + list(importer.normalize_task(task, "test_task")) + task = _task() + task["actions"][0]["cleaned_html"] = "" + with self.assertRaisesRegex(ValueError, "missing from cleaned DOM"): + list(importer.normalize_task(task, "test_task")) + + def test_duplicate_dom_node_and_action_ids_fail_closed(self): + task = _task() + task["actions"][0]["cleaned_html"] = task["actions"][0]["cleaned_html"].replace( + "", '' + ) + with self.assertRaisesRegex(ValueError, "duplicate candidate node"): + list(importer.normalize_task(task, "test_task")) + task = _task() + task["actions"][1]["action_uid"] = task["actions"][0]["action_uid"] + with self.assertRaisesRegex(ValueError, "duplicate action"): + list(importer.normalize_task(task, "test_task")) + + def test_unknown_or_conflicting_splits_fail_closed(self): + with self.assertRaisesRegex(ValueError, "official held-out"): + list(importer.normalize_task(_task(), "train")) + task = _task() + task["split"] = "test_domain" + with self.assertRaisesRegex(ValueError, "disagrees"): + list(importer.normalize_task(task, "test_task")) + + def test_history_misalignment_fails_closed(self): + task = _task() + task["action_reprs"] = ["ONLY_ONE"] + with self.assertRaisesRegex(ValueError, "aligned"): + list(importer.normalize_task(task, "test_task")) + + def test_streaming_json_survives_small_chunk_boundaries(self): + tasks = [_task(), _task("task-b", "action-b")] + with patch.object(importer, "CHUNK_SIZE", 17): + self.assertEqual(list(importer.iter_tasks(io.StringIO(json.dumps(tasks)))), tasks) + self.assertEqual(list(importer.iter_tasks(io.StringIO(" [ ] \n"))), []) + + def test_streaming_large_task_reads_grow_without_changing_task_boundaries(self): + class RecordingReader(io.StringIO): + def __init__(self, value): + super().__init__(value) + self.read_sizes = [] + + def read(self, size=-1): + self.read_sizes.append(size) + return super().read(size) + + tasks = [_task(), _task("second-task", "second-action")] + tasks[0]["actions"][0]["raw_html"] = "unneeded source text " * 500 + reader = RecordingReader(json.dumps(tasks)) + with patch.object(importer, "CHUNK_SIZE", 32): + self.assertEqual(list(importer.iter_tasks(reader)), tasks) + self.assertLess(len(reader.read_sizes), 32) + + def test_streaming_json_rejects_malformed_and_oversized_inputs(self): + for source in ("{}", "[{},]", "[{}", "[{}]{}", "[{]", "[,{}]"): + with self.subTest(source=source), self.assertRaises(ValueError): + list(importer.iter_tasks(io.StringIO(source))) + with ( + patch.object(importer, "MAX_TASK_CHARS", 20), + self.assertRaisesRegex(ValueError, "buffer bound"), + ): + list(importer.iter_tasks(io.StringIO(json.dumps([_task()])))) + + def test_archive_guards_reject_traversal_unknown_and_duplicate_members(self): + for name in ( + "../test_task_0.json", + "/test_task/test_task_0.json", + "test_task/../../escape.json", + "test_task\\test_task_0.json", + "train/train_0.json", + "test_task//test_task_0.json", + ): + with self.subTest(name=name), self.assertRaises(ValueError): + importer.validate_archive_members([zipfile.ZipInfo(name)]) + info = zipfile.ZipInfo("test_task/test_task_0.json") + info.file_size = 1 + with self.assertRaisesRegex(ValueError, "duplicate"): + importer.validate_archive_members([info, info]) + + def test_archive_guards_require_exact_members_and_bounded_regular_files(self): + infos = [] + for split, count in importer.OFFICIAL_FILE_COUNTS.items(): + for index in range(count): + info = zipfile.ZipInfo(f"{split}/{split}_{index}.json") + info.file_size = 1 + infos.append(info) + importer.validate_archive_members(infos) + with self.assertRaisesRegex(ValueError, "member set"): + importer.validate_archive_members(infos[:-1]) + infos[0].external_attr = (stat.S_IFLNK | 0o777) << 16 + with self.assertRaisesRegex(ValueError, "symlink"): + importer.validate_archive_members(infos) + infos[0].external_attr = 0 + infos[0].file_size = importer.MAX_MEMBER_BYTES + 1 + with self.assertRaisesRegex(ValueError, "size bound"): + importer.validate_archive_members(infos) + + def _fixture_files(self, root): + input_dir = root / "input" + manifest = _manifest() + for index, split in enumerate(importer.SPLITS): + name = f"{split}/{split}_0.json" + path = input_dir / name + path.parent.mkdir(parents=True) + path.write_text(json.dumps([_task(f"task-{index}", f"action-{index}")])) + manifest["files"].append( + { + "path": name, + "split": split, + "bytes": path.stat().st_size, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + ) + manifest_path = root / "source.json" + manifest_path.write_text(json.dumps(manifest)) + return input_dir, manifest_path + + def test_import_outputs_provenance_and_contract_compatible_jsonl(self): + # Fixture counts are patched locally; the production CLI has no count override. + with ( + TemporaryDirectory() as directory, + patch.object(importer, "OFFICIAL_FILE_COUNTS", dict.fromkeys(importer.SPLITS, 1)), + patch.object(importer, "OFFICIAL_TASK_COUNTS", dict.fromkeys(importer.SPLITS, 1)), + ): + root = Path(directory) + input_dir, manifest_path = self._fixture_files(root) + output = root / "output" + report = importer.import_split(input_dir, importer.SPLITS, output, manifest_path) + self.assertEqual(report["task_counts"], dict.fromkeys(importer.SPLITS, 1)) + self.assertEqual(report["row_counts"], dict.fromkeys(importer.SPLITS, 2)) + for line in (output / "rows.jsonl").read_text().splitlines(): + row = json.loads(line) + Mind2WebExample.from_mapping(row) + self.assertEqual(row["provenance"]["dataset_revision"], importer.DATASET_REVISION) + for record in report["outputs"]: + self.assertEqual( + record["sha256"], + hashlib.sha256((output / record["path"]).read_bytes()).hexdigest(), + ) + with self.assertRaisesRegex(ValueError, "overwrite"): + importer.import_split(input_dir, importer.SPLITS, output, manifest_path) + + def test_tampering_and_incomplete_task_counts_create_no_output(self): + with ( + TemporaryDirectory() as directory, + patch.object(importer, "OFFICIAL_FILE_COUNTS", dict.fromkeys(importer.SPLITS, 1)), + ): + root = Path(directory) + input_dir, manifest_path = self._fixture_files(root) + output = root / "output" + with self.assertRaisesRegex(ValueError, "task counts"): + importer.import_split(input_dir, importer.SPLITS, output, manifest_path) + self.assertFalse(output.exists()) + path = input_dir / "test_task/test_task_0.json" + path.write_text(path.read_text().replace("blue", "pink")) + with self.assertRaisesRegex(ValueError, "hash verification"): + importer.import_split(input_dir, importer.SPLITS, output, manifest_path) + self.assertFalse(output.exists()) + + def test_duplicate_tasks_across_splits_fail_closed(self): + with ( + TemporaryDirectory() as directory, + patch.object(importer, "OFFICIAL_FILE_COUNTS", dict.fromkeys(importer.SPLITS, 1)), + ): + root = Path(directory) + input_dir, manifest_path = self._fixture_files(root) + manifest = json.loads(manifest_path.read_text()) + for record in manifest["files"]: + path = input_dir / record["path"] + path.write_text(json.dumps([_task()])) + record["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + record["bytes"] = path.stat().st_size + manifest_path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, "duplicate task"): + importer.import_split(input_dir, importer.SPLITS, root / "output", manifest_path) + + def test_manifest_rejects_unknown_revision_and_training_usage(self): + manifest = _manifest() + importer._validate_manifest(manifest) + manifest["dataset_revision"] = "0" * 40 + with self.assertRaisesRegex(ValueError, "pinned"): + importer._validate_manifest(manifest) + manifest = _manifest() + manifest["usage"] = "training" + with self.assertRaisesRegex(ValueError, "evaluation-only"): + importer._validate_manifest(manifest) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mind2web_manifest.py b/tests/test_mind2web_manifest.py new file mode 100644 index 0000000000000000000000000000000000000000..ba9cac6c6e2f4b645ae6bec29c807fce02b1b47a --- /dev/null +++ b/tests/test_mind2web_manifest.py @@ -0,0 +1,59 @@ +import hashlib +import importlib.util +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + + +def _load_builder(): + path = Path(__file__).parents[1] / "tools" / "build_mind2web_manifest.py" + spec = importlib.util.spec_from_file_location("vons_mind2web_manifest", path) + if spec is None or spec.loader is None: + raise RuntimeError("could not load Mind2Web manifest builder") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +builder = _load_builder() + + +class Mind2WebManifestTests(unittest.TestCase): + def test_manifest_hashes_splits_without_public_paths(self) -> None: + with TemporaryDirectory() as directory: + split_path = Path(directory) / "cross-task.jsonl" + payload = b'{"id":"a"}\n' + split_path.write_bytes(payload) + manifest = builder.build_manifest( + variant="original-mind2web", + source_revision="rev-1", + permitted_use="evaluation_only", + approval_record="owner-approved-2026-09-24", + preprocessing_revision="prep-1", + retriever_revision="retriever-1", + splits=[("cross-task", split_path)], + ) + + self.assertEqual(manifest["schema"], "vons.mind2web-provenance/v1") + self.assertEqual(manifest["splits"][0]["bytes"], len(payload)) + self.assertEqual(manifest["splits"][0]["sha256"], hashlib.sha256(payload).hexdigest()) + self.assertEqual(manifest["raw_data_paths"], "omitted from public artifacts") + + def test_manifest_rejects_duplicate_split_names(self) -> None: + with TemporaryDirectory() as directory: + path = Path(directory) / "split.jsonl" + path.write_text("{}\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "unique"): + builder.build_manifest( + variant="original-mind2web", + source_revision="rev-1", + permitted_use="evaluation_only", + approval_record="approved", + preprocessing_revision="prep-1", + retriever_revision="retriever-1", + splits=[("test", path), ("test", path)], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mind2web_prediction.py b/tests/test_mind2web_prediction.py new file mode 100644 index 0000000000000000000000000000000000000000..61cbe150f97d78d0694dd04557b6ea990e3f61bd --- /dev/null +++ b/tests/test_mind2web_prediction.py @@ -0,0 +1,86 @@ +import importlib.util +import json +from pathlib import Path + +import pytest + +spec = importlib.util.spec_from_file_location("predict_mind2web", Path(__file__).parents[1] / "tools/predict_mind2web.py") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def row(identifier="a"): + candidates = [{"id": str(i), "text": f"Element {i}"} for i in range(32)] + return {"id": identifier, "task_id": "task", "split": "test_task", + "state": {"goal": "select"}, "candidates": candidates, + "generated_candidates": [item["id"] for item in candidates], + "input_sha256": "a" * 64} + + +def test_exact_k_and_label_free_contract(): + assert len(module.make_request(row(), 5)["candidates"]) == 5 + assert len(module.make_request(row(), 32)["candidates"]) == 32 + with pytest.raises(ValueError, match="label-free"): + module.make_request({**row(), "positive_ids": ["0"]}, 5) + bad = row() + bad["generated_candidates"].reverse() + with pytest.raises(ValueError, match="order"): + module.make_request(bad, 5) + missing = row() + del missing["input_sha256"] + with pytest.raises(ValueError, match="input_sha256 before inference"): + module.make_request(missing, 5) + + +def test_decode_abstention_finite_and_ids(): + result = module.decode([0, 10], 10, ["first", "second"]) + assert result["selection"] == "second" + assert sum(result["probabilities"].values()) == pytest.approx(1) + assert module.decode([0, 10], -10, ["first", "second"])["selection"] is None + with pytest.raises(ValueError, match="invalid scores"): + module.decode([float("nan"), 1], 10, ["first", "second"]) + with pytest.raises(module.ModelOutputError, match="invalid scores"): + module.decode([-float("inf"), -float("inf")], 10, ["first", "second"]) + + +def test_runner_invokes_each_k_and_preserves_errors(tmp_path): + source = tmp_path / "retrieval.jsonl" + source.write_text("\n".join(json.dumps(row(identifier)) for identifier in ["a", "b"])) + calls = [] + + def select(request): + calls.append(len(request["candidates"])) + if len(calls) % 2 == 0: + raise ValueError("input_overflow") + return {"selection": request["candidates"][0]["id"], "status": "ok"} + + for k in (5, 10): + output = tmp_path / f"k{k}.jsonl" + report = module.predict_file(source, output, split="test_task", k=k, + select=select, provenance={"backend": "test"}) + predictions = [json.loads(line) for line in output.read_text().splitlines()] + assert len(predictions) == 2 + assert predictions[1]["selection"] is None + assert report["counts"] == {"ok": 1, "error": 1} + assert not report["selection_reused_across_k"] + assert all(len(item["generated_candidates"]) == k for item in predictions) + with pytest.raises(ValueError, match="overwrite"): + module.predict_file(source, output, split="test_task", k=k, + select=select, provenance={}) + assert calls == [5, 5, 10, 10] + + +def test_nonfinite_model_output_stops_comparison_and_keeps_journal(tmp_path): + source = tmp_path / "retrieval.jsonl" + source.write_text(json.dumps(row())) + output = tmp_path / "predictions.jsonl" + + def invalid(request): + raise module.ModelOutputError("non-finite output") + + with pytest.raises(module.ModelOutputError): + module.predict_file(source, output, split="test_task", k=5, + select=invalid, provenance={}) + assert not output.exists() + assert output.with_suffix(".jsonl.partial").exists() + assert not output.with_suffix(".manifest.json").exists() diff --git a/tests/test_mind2web_replay.py b/tests/test_mind2web_replay.py new file mode 100644 index 0000000000000000000000000000000000000000..af3f02102aa51269ee52402b464bcfcfe40d11fc --- /dev/null +++ b/tests/test_mind2web_replay.py @@ -0,0 +1,392 @@ +"""Small fabricated inputs only; no model or official-corpus replay in tests.""" + +import csv +import hashlib +import importlib.util +import json +import unittest +from collections import Counter +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + + +def _load_replay(): + path = Path(__file__).parents[1] / "tools/replay_mind2web_evaluation.py" + spec = importlib.util.spec_from_file_location("vons_mind2web_replay", path) + if spec is None or spec.loader is None: + raise RuntimeError("could not load replay tool") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +replay = _load_replay() + + +def _write(path, value): + path.write_text(json.dumps(value), encoding="utf-8") + + +def _write_rows(path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + + +def _read_rows(path): + return [json.loads(line) for line in path.read_text().splitlines()] + + +def _fixture(root, cells=(("test_task", 5),)): + rows_path = root / "rows.jsonl" + index_path = root / "run-index.json" + predictions_dir = root / "predictions" + predictions_dir.mkdir() + gold = [] + for split in dict.fromkeys(split for split, _ in cells): + for suffix in ("accepted", "abstained", "errored", "missing"): + gold.append( + { + "id": f"{split}-{suffix}", + "task_id": f"{split}-task-{suffix}", + "action_id": suffix, + "split": split, + "candidate_ids": ["x", "y"], + "positive_ids": ["x"], + } + ) + _write_rows(rows_path, gold) + runner_hash = replay._hash(replay._source_paths()["tools/predict_mind2web.py"]) + index = {"source_sha256": "e" * 64, "runner_sha256": runner_hash, "runs": []} + for split, k in cells: + config = { + "split": split, + "k": k, + "provenance": {"backend": "fixture"}, + "adapted": False, + "question": "Fixture question", + } + config_hash = replay._object_hash(config) + predictions = [] + for suffix, status, reason in ( + ("accepted", "ok", None), + ("abstained", "abstain", "confidence_below_threshold"), + ("errored", "error", "input_overflow"), + ): + predictions.append( + { + "id": f"{split}-{suffix}", + "task_id": f"{split}-task-{suffix}", + "split": split, + "k": k, + "generated_candidates": ["x", "y"], + "selection": "x" if status == "ok" else None, + "status": status, + "reason": reason, + "config_sha256": config_hash, + "request_sha256": "a" * 64, + "retrieval_input_sha256": "b" * 64, + } + ) + prediction_path = predictions_dir / f"{split}-k{k}.jsonl" + _write_rows(prediction_path, predictions) + counts = dict(Counter(row["status"] for row in predictions)) + digest = replay._hash(prediction_path) + manifest = { + "schema": "vons.mind2web-prediction/v1", + "config": config, + "config_sha256": config_hash, + "source_sha256": index["source_sha256"], + "runner_sha256": runner_hash, + "predictions_sha256": digest, + "rows": len(predictions), + "counts": counts, + "selection_reused_across_k": False, + } + _write(prediction_path.with_suffix(".manifest.json"), manifest) + index["runs"].append( + { + "split": split, + "k": k, + "rows": len(predictions), + "counts": counts, + "predictions_sha256": digest, + } + ) + _write(index_path, index) + return rows_path, index_path, predictions_dir, root / "output" + + +def _refresh_prediction_hashes(index_path, prediction_path): + digest = replay._hash(prediction_path) + manifest_path = prediction_path.with_suffix(".manifest.json") + manifest = json.loads(manifest_path.read_text()) + manifest["predictions_sha256"] = digest + _write(manifest_path, manifest) + index = json.loads(index_path.read_text()) + for run in index["runs"]: + if prediction_path.name == f"{run['split']}-k{run['k']}.jsonl": + run["predictions_sha256"] = digest + _write(index_path, index) + + +class Mind2WebReplayTests(unittest.TestCase): + def test_observed_outcomes_missing_denominator_and_real_bootstrap(self): + with TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + summary = replay.replay(*paths, allow_subset=True) + run = summary["runs"][0] + self.assertEqual(run["counts"], {"ok": 1, "abstain": 1, "error": 1}) + self.assertEqual( + run["execution_outcomes"], {"accepted": 1, "error": 1, "abstain": 1, "missing": 1} + ) + self.assertEqual(run["acceptance_coverage"], 0.25) + self.assertEqual(run["prediction_coverage"], 0.75) + self.assertEqual(run["coverage_denominator_rows"], 4) + self.assertEqual(run["missing_prediction_record_count"], 1) + self.assertEqual( + run["execution_failure_counts"], + {"confidence_below_threshold": 1, "input_overflow": 1}, + ) + self.assertIn({"reason": None, "count": 1}, run["execution_reason_counts"]) + self.assertEqual(run["metrics"]["correct_selections"], 1) + self.assertEqual(run["metrics"]["missing_predictions"], 2) + self.assertEqual(run["bootstrap"]["draws"], 1000) + self.assertEqual(run["bootstrap"]["seed"], 7) + self.assertFalse(summary["matrix"]["complete"]) + output = paths[-1] + report_path = output / "test_task-k5-metrics.json" + self.assertEqual(run["sha256"], hashlib.sha256(report_path.read_bytes()).hexdigest()) + report = json.loads(report_path.read_text()) + self.assertEqual(report["input_sha256"]["rows"], replay._hash(paths[0])) + self.assertEqual(report["source_sha256"], summary["source_sha256"]) + self.assertEqual(summary["input_sha256"]["run_index"], replay._hash(paths[1])) + table_path = output / "summary.csv" + with table_path.open(newline="") as handle: + table_row = next(csv.DictReader(handle)) + self.assertEqual(table_row["coverage"], "0.25") + self.assertEqual(table_row["missing_count"], "1") + self.assertEqual(table_row["error_count"], "1") + self.assertEqual(summary["table"]["sha256"], replay._hash(table_path)) + + def test_undefined_metric_cells_remain_blank_in_csv_and_null_in_json(self): + with TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + rows = _read_rows(paths[0]) + for row in rows: + row["positive_ids"] = [] + row["no_positive"] = True + _write_rows(paths[0], rows) + summary = replay.replay(*paths, allow_subset=True) + self.assertIsNone(summary["runs"][0]["metrics"]["candidate_recall_step_micro"]) + with (paths[-1] / "summary.csv").open(newline="") as handle: + row = next(csv.DictReader(handle)) + for field in ( + "recall_micro", + "recall_task_macro", + "taskmacroCIlo", + "taskmacroCIhi", + "conditional_accuracy_micro", + "conditional_accuracy_task_macro", + ): + self.assertEqual(row[field], "") + self.assertEqual(row["positive_rows"], "0") + + def test_default_requires_complete_matrix_and_tiny_twelve_cells_succeed(self): + with TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + with self.assertRaisesRegex(ValueError, "all 12"): + replay.replay(*paths) + self.assertFalse(paths[-1].exists()) + with TemporaryDirectory() as directory: + paths = _fixture(Path(directory), replay.CELLS) + summary = replay.replay(*paths) + self.assertTrue(summary["matrix"]["complete"]) + self.assertEqual(len(summary["runs"]), 12) + self.assertEqual(len(list(paths[-1].glob("*-metrics.json"))), 12) + + def test_duplicate_unknown_and_bool_cells_fail_closed(self): + for mutation in ("duplicate", "unknown_split", "unknown_k", "bool_k"): + with self.subTest(mutation=mutation), TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + index = json.loads(paths[1].read_text()) + if mutation == "duplicate": + index["runs"].append(index["runs"][0]) + elif mutation == "unknown_split": + index["runs"][0]["split"] = "train" + else: + index["runs"][0]["k"] = True if mutation == "bool_k" else 8 + _write(paths[1], index) + with self.assertRaises(ValueError): + replay.replay(*paths, allow_subset=True) + self.assertFalse(paths[-1].exists()) + + def test_tampered_predictions_fail_index_hash_check(self): + with TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + prediction_path = paths[2] / "test_task-k5.jsonl" + with prediction_path.open("a") as handle: + handle.write("\n") + with self.assertRaisesRegex(ValueError, "prediction file hash"): + replay.replay(*paths, allow_subset=True) + + def test_row_count_and_raw_status_claims_are_checked(self): + for field in ("rows", "counts"): + with self.subTest(field=field), TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + index = json.loads(paths[1].read_text()) + index["runs"][0][field] = 4 if field == "rows" else {"ok": 3} + _write(paths[1], index) + with self.assertRaisesRegex(ValueError, "row count|raw status counts"): + replay.replay(*paths, allow_subset=True) + + def test_manifest_config_and_source_identity_hashes_are_checked(self): + for field in ("config_split", "config_hash", "source_hash", "runner_hash", "reuse"): + with self.subTest(field=field), TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + manifest_path = paths[2] / "test_task-k5.manifest.json" + manifest = json.loads(manifest_path.read_text()) + if field == "config_split": + manifest["config"]["split"] = "test_domain" + elif field == "config_hash": + manifest["config_sha256"] = "0" * 64 + elif field == "source_hash": + manifest["source_sha256"] = "0" * 64 + elif field == "runner_hash": + manifest["runner_sha256"] = "0" * 64 + else: + manifest["selection_reused_across_k"] = True + _write(manifest_path, manifest) + with self.assertRaises(ValueError): + replay.replay(*paths, allow_subset=True) + + def test_prediction_row_identity_config_and_status_consistency(self): + mutations = { + "split": "test_domain", + "k": 10, + "task_id": "another-task", + "config_sha256": "0" * 64, + "request_sha256": "bad", + "status": "error", + "selection": "outside", + "generated_candidates": ["outside"], + } + for key, value in mutations.items(): + with self.subTest(key=key), TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + prediction_path = paths[2] / "test_task-k5.jsonl" + rows = _read_rows(prediction_path) + rows[0][key] = value + _write_rows(prediction_path, rows) + _refresh_prediction_hashes(paths[1], prediction_path) + with self.assertRaises(ValueError): + replay.replay(*paths, allow_subset=True) + + def test_unknown_or_duplicate_prediction_ids_fail_closed(self): + for duplicate in (False, True): + with self.subTest(duplicate=duplicate), TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + prediction_path = paths[2] / "test_task-k5.jsonl" + rows = _read_rows(prediction_path) + rows[1]["id"] = rows[0]["id"] if duplicate else "unknown" + _write_rows(prediction_path, rows) + _refresh_prediction_hashes(paths[1], prediction_path) + with self.assertRaisesRegex(ValueError, "unknown or duplicated"): + replay.replay(*paths, allow_subset=True) + + def test_missing_reason_is_counted_without_inventing_a_failure_reason(self): + with TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + prediction_path = paths[2] / "test_task-k5.jsonl" + rows = _read_rows(prediction_path) + del rows[-1]["reason"] + _write_rows(prediction_path, rows) + _refresh_prediction_hashes(paths[1], prediction_path) + run = replay.replay(*paths, allow_subset=True)["runs"][0] + self.assertEqual(run["missing_reason_fields"], 1) + self.assertNotIn("input_overflow", run["execution_failure_counts"]) + self.assertIn({"reason": None, "count": 2}, run["execution_reason_counts"]) + + def test_replay_refuses_existing_evidence(self): + with TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + paths[-1].mkdir() + sentinel = paths[-1] / "keep.txt" + sentinel.write_text("original") + with self.assertRaisesRegex(ValueError, "overwrite"): + replay.replay(*paths, allow_subset=True) + self.assertEqual(sentinel.read_text(), "original") + + def test_input_mutation_during_evaluation_discards_all_staged_reports(self): + for which in ("gold", "prediction", "manifest", "index"): + with self.subTest(which=which), TemporaryDirectory() as directory: + paths = _fixture(Path(directory)) + evaluator = replay._load_evaluator( + replay._source_paths()["tools/evaluate_mind2web.py"] + ) + evaluate = evaluator.evaluate_files + targets = { + "gold": paths[0], + "index": paths[1], + "prediction": paths[2] / "test_task-k5.jsonl", + "manifest": paths[2] / "test_task-k5.manifest.json", + } + + def mutate(*args, evaluate=evaluate, target=targets[which], **kwargs): + report = evaluate(*args, **kwargs) + with target.open("a") as handle: + handle.write("\n") + return report + + evaluator.evaluate_files = mutate + with ( + patch.object(replay, "_load_evaluator", return_value=evaluator), + self.assertRaisesRegex(ValueError, "source mutation"), + ): + replay.replay(*paths, allow_subset=True) + self.assertFalse(paths[-1].exists()) + + def test_code_mutation_during_replay_is_rejected(self): + with TemporaryDirectory() as directory: + root = Path(directory) + paths = _fixture(root) + sources = replay._source_paths() + stand_in = root / "source-snapshot.py" + stand_in.write_text("# Initial source snapshot\n") + sources["tools/replay_mind2web_evaluation.py"] = stand_in + evaluator = replay._load_evaluator(sources["tools/evaluate_mind2web.py"]) + evaluate = evaluator.evaluate_files + + def mutate(*args, **kwargs): + report = evaluate(*args, **kwargs) + stand_in.write_text("# Changed during replay\n") + return report + + evaluator.evaluate_files = mutate + with ( + patch.object(replay, "_source_paths", return_value=sources), + patch.object(replay, "_load_evaluator", return_value=evaluator), + self.assertRaisesRegex(ValueError, "source mutation"), + ): + replay.replay(*paths, allow_subset=True) + self.assertFalse(paths[-1].exists()) + + def test_incompatible_cell_configurations_are_rejected(self): + with TemporaryDirectory() as directory: + paths = _fixture(Path(directory), (("test_task", 5), ("test_task", 10))) + prediction_path = paths[2] / "test_task-k10.jsonl" + manifest_path = prediction_path.with_suffix(".manifest.json") + manifest = json.loads(manifest_path.read_text()) + manifest["config"]["provenance"]["backend"] = "different-model" + manifest["config_sha256"] = replay._object_hash(manifest["config"]) + _write(manifest_path, manifest) + rows = _read_rows(prediction_path) + for row in rows: + row["config_sha256"] = manifest["config_sha256"] + _write_rows(prediction_path, rows) + _refresh_prediction_hashes(paths[1], prediction_path) + with self.assertRaisesRegex(ValueError, "incompatible frozen"): + replay.replay(*paths, allow_subset=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mind2web_runner.py b/tests/test_mind2web_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..4101d10eaa5e3e2a3a1e27ae3a0a972c731999d6 --- /dev/null +++ b/tests/test_mind2web_runner.py @@ -0,0 +1,149 @@ +import importlib.util +import json +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + + +def _load_runner(): + path = Path(__file__).parents[1] / "tools" / "evaluate_mind2web.py" + spec = importlib.util.spec_from_file_location("vons_mind2web_runner", path) + if spec is None or spec.loader is None: + raise RuntimeError("could not load Mind2Web runner") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +runner = _load_runner() + + +class Mind2WebRunnerTests(unittest.TestCase): + def test_runner_preserves_missing_prediction_rows(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + rows_path = root / "rows.jsonl" + predictions_path = root / "predictions.jsonl" + rows_path.write_text( + "\n".join( + [ + json.dumps({"id": "a", "candidate_ids": [], "positive_ids": ["target"]}), + json.dumps({"id": "b", "candidate_ids": [], "positive_ids": ["target"]}), + ] + ) + + "\n", + encoding="utf-8", + ) + predictions_path.write_text( + json.dumps( + {"id": "a", "generated_candidates": ["target"], "selection": "target"} + ) + + "\n", + encoding="utf-8", + ) + + report = runner.evaluate_files(rows_path, predictions_path, [1, 5]) + + self.assertEqual(report["rows"], 2) + self.assertEqual(report["prediction_rows"], 1) + self.assertEqual(report["metrics_by_k"][0]["candidate_recall"], 0.5) + self.assertIsNone(report["metrics_by_k"][0]["selection_accuracy_given_recall"]) + self.assertFalse(report["selection_evaluation"]["selection_metrics_valid"]) + self.assertEqual(len(report["input_sha256"]["rows"]), 64) + self.assertEqual(len(report["input_sha256"]["predictions"]), 64) + self.assertIn("task_bootstrap_by_k", report) + + single_k_report = runner.evaluate_files(rows_path, predictions_path, [1]) + self.assertEqual(single_k_report["metrics_by_k"][0]["selection_accuracy_given_recall"], 1.0) + + def test_runner_requires_prediction_k_to_match_single_requested_k(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + rows_path = root / "rows.jsonl" + predictions_path = root / "predictions.jsonl" + rows_path.write_text( + json.dumps({"id": "a", "candidate_ids": [], "positive_ids": ["target"]}) + "\n", + encoding="utf-8", + ) + predictions_path.write_text( + json.dumps({"id": "a", "k": 5, "generated_candidates": ["target"], "selection": "target"}) + + "\n", + encoding="utf-8", + ) + + with self.assertRaisesRegex(ValueError, "exactly that single k"): + runner.evaluate_files(rows_path, predictions_path, [10]) + + def test_runner_filters_normalized_rows_by_split_before_matching(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + rows_path = root / "rows.jsonl" + predictions_path = root / "predictions.jsonl" + rows_path.write_text( + "\n".join( + [ + json.dumps({"id": "keep", "split": "test_task", "candidate_ids": [], "positive_ids": ["target"]}), + json.dumps({"id": "drop", "split": "test_domain", "candidate_ids": [], "positive_ids": ["target"]}), + ] + ) + + "\n", + encoding="utf-8", + ) + predictions_path.write_text( + json.dumps({"id": "keep", "generated_candidates": ["target"], "selection": "target"}) + + "\n", + encoding="utf-8", + ) + + report = runner.evaluate_files(rows_path, predictions_path, [1], split="test_task") + + self.assertEqual(report["split"], "test_task") + self.assertEqual(report["rows"], 1) + self.assertEqual(report["metrics_by_k"][0]["candidate_recall"], 1.0) + + def test_runner_rejects_unknown_prediction_ids(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + rows_path = root / "rows.jsonl" + predictions_path = root / "predictions.jsonl" + rows_path.write_text( + json.dumps({"id": "a", "candidate_ids": [], "positive_ids": ["target"]}) + "\n", + encoding="utf-8", + ) + predictions_path.write_text( + json.dumps({"id": "unknown", "generated_candidates": []}) + "\n", + encoding="utf-8", + ) + + with self.assertRaisesRegex(ValueError, "unknown example ids"): + runner.evaluate_files(rows_path, predictions_path, [5]) + + def test_runner_rejects_duplicate_gold_ids_and_missing_generated_candidates(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + rows_path = root / "rows.jsonl" + predictions_path = root / "predictions.jsonl" + rows_path.write_text( + "\n".join( + [ + json.dumps({"id": "a", "candidate_ids": [], "positive_ids": ["target"]}), + json.dumps({"id": "a", "candidate_ids": [], "positive_ids": ["target"]}), + ] + ) + + "\n", + encoding="utf-8", + ) + predictions_path.write_text(json.dumps({"id": "a"}) + "\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "duplicate gold"): + runner.evaluate_files(rows_path, predictions_path, [5]) + + rows_path.write_text( + json.dumps({"id": "a", "candidate_ids": [], "positive_ids": ["target"]}) + "\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "generated_candidates"): + runner.evaluate_files(rows_path, predictions_path, [5]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_public_release.py b/tests/test_public_release.py new file mode 100644 index 0000000000000000000000000000000000000000..01263a00e0541803908340e0af57772845b0a014 --- /dev/null +++ b/tests/test_public_release.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import subprocess +from pathlib import Path + +import pytest + +_SPEC = importlib.util.spec_from_file_location( + "prepare_public_release", Path(__file__).parents[1] / "tools/prepare_public_release.py" +) +assert _SPEC and _SPEC.loader +_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_MODULE) +inspect_file = _MODULE.inspect_file +prepare = _MODULE.prepare + + +def source_tree(tmp_path: Path) -> Path: + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / "configs").mkdir() + (tmp_path / "docs").mkdir() + (tmp_path / "README.md").write_text("# Source\n") + (tmp_path / "docs/HUGGING_FACE.md").write_text( + "# Hub source preview\n[Model card](../docs/MODEL_CARD.md)\n" + ) + (tmp_path / ".gitignore").write_text("data/\nreleases/\n.env\n") + spec = {"license_status": "pending", "files": [ + ".gitignore", "README.md", "configs/public-release.json", "docs/HUGGING_FACE.md", + ]} + (tmp_path / "configs/public-release.json").write_text(json.dumps(spec)) + return tmp_path + + +def test_export_omits_private_data_and_replaces_hub_card(tmp_path: Path) -> None: + root = source_tree(tmp_path) + (root / "data").mkdir() + (root / "data/secret.jsonl").write_text("private row") + (root / ".env").write_text("SECRET=private") + output = root / "releases/hub" + manifest = prepare(root, output, "huggingface") + assert (output / "README.md").read_text() == ( + "# Hub source preview\n[Model card](docs/MODEL_CARD.md)\n" + ) + assert (output / "docs/HUGGING_FACE.md").read_text() == ( + root / "docs/HUGGING_FACE.md" + ).read_text() + assert not (output / "data").exists() + assert not (output / ".env").exists() + for entry in manifest["files"]: + assert hashlib.sha256((output / entry["path"]).read_bytes()).hexdigest() == entry["sha256"] + assert (root / "data/secret.jsonl").read_text() == "private row" + with pytest.raises(ValueError, match="overwrite"): + prepare(root, output, "huggingface") + + +def test_unreviewed_and_force_tracked_files_block_export(tmp_path: Path) -> None: + root = source_tree(tmp_path) + (root / ".env").write_text("PRIVATE=value") + subprocess.run(["git", "add", "-f", ".env"], cwd=root, check=True) + with pytest.raises(ValueError, match="unexpected_public_paths"): + prepare(root) + + +def test_new_public_file_requires_review(tmp_path: Path) -> None: + root = source_tree(tmp_path) + (root / "notes.md").write_text("unreviewed") + with pytest.raises(ValueError, match="unexpected_public_paths"): + prepare(root) + + +@pytest.mark.parametrize("name", ["../outside.md", "/outside.md", "data/example.md", ".env"]) +def test_unsafe_allowlist_path_is_rejected(tmp_path: Path, name: str) -> None: + with pytest.raises(ValueError, match="disallowed path"): + inspect_file(tmp_path, name) + + +def test_symlink_cannot_escape_export(tmp_path: Path) -> None: + (tmp_path / "original.md").write_text("private") + (tmp_path / "link.md").symlink_to(tmp_path / "original.md") + with pytest.raises(ValueError, match="symlink"): + inspect_file(tmp_path, "link.md") + + +def test_secret_finding_does_not_echo_the_value(tmp_path: Path) -> None: + token = "hf_" + "a" * 30 + (tmp_path / "source.py").write_text(f'TOKEN = "{token}"\n') + with pytest.raises(ValueError, match="provider-token") as failure: + inspect_file(tmp_path, "source.py") + assert token not in str(failure.value) + + +def test_publication_binary_is_bound_to_reviewed_digest(tmp_path: Path) -> None: + path = tmp_path / "docs/publications/paper.pdf" + path.parent.mkdir(parents=True) + path.write_bytes(b"%PDF-example") + name = path.relative_to(tmp_path).as_posix() + with pytest.raises(ValueError, match="reviewed digest"): + inspect_file(tmp_path, name) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + assert inspect_file(tmp_path, name, digest) == path.read_bytes() + path.write_bytes(b"%PDF-changed") + with pytest.raises(ValueError, match="reviewed digest"): + inspect_file(tmp_path, name, digest) diff --git a/tests/test_quality_summary.py b/tests/test_quality_summary.py new file mode 100644 index 0000000000000000000000000000000000000000..c98ed1c60434289bb8c47e0bfd06087ceed170d9 --- /dev/null +++ b/tests/test_quality_summary.py @@ -0,0 +1,88 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path + +from vons.data import smoke_examples, write_jsonl +from vons.evaluation import Prediction, write_report + + +def _load_summary_tool(): + path = Path(__file__).parents[1] / "tools" / "summarize_quality.py" + spec = importlib.util.spec_from_file_location("vons_quality_summary", path) + if spec is None or spec.loader is None: + raise RuntimeError("could not load quality summary tool") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +summary_tool = _load_summary_tool() + + +class QualitySummaryTests(unittest.TestCase): + def test_recomputes_saved_direct_and_diffusion_quality(self) -> None: + root = Path(__file__).parents[1] + data_path = root / "data/generated/pilot-test.jsonl" + reports = [ + root / "reports/pilot-direct-test.json", + root / "reports/pilot-diffusion-test.json", + ] + if not all(path.is_file() for path in [data_path, *reports]): + self.skipTest("Historical pilot inputs are intentionally excluded from public source") + result = summary_tool.build_quality_summary(data_path, reports) + + self.assertEqual(result["measurement_label"], "measured_synthetic_pilot") + self.assertEqual(result["dataset"]["rows"], 200) + self.assertEqual(result["reports"][0]["metrics"]["accuracy_answerable"], 1.0) + self.assertEqual(result["reports"][1]["metrics"]["coverage"], 0.155) + self.assertIsNone(result["external_evaluation"]["mind2web"]["candidate_recall"]) + self.assertEqual(result["external_evaluation"]["mind2web"]["status"], "not_evaluated") + + def test_recomputes_metrics_without_private_pilot_files(self) -> None: + rows = smoke_examples()[:2] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + data_path = root / "synthetic.jsonl" + write_jsonl(data_path, rows) + reports = [] + for backend in ("direct", "diffusion"): + predictions = [ + Prediction( + row.id, row.label, + {option: float(option == row.label) for option in row.options}, + 1.0, backend == "diffusion" and index == 1, + ) + for index, row in enumerate(rows) + ] + report_path = root / f"{backend}.json" + write_report(report_path, config={"backend": backend, "seed": 7}, + summary={"coverage": -1}, predictions=predictions) + reports.append(report_path) + result = summary_tool.build_quality_summary(data_path, reports, version="v2") + + self.assertEqual(result["dataset"]["rows"], 2) + self.assertEqual(result["metric_version"], "vons.metrics/v2") + direct, diffusion = result["reports"] + self.assertEqual(direct["metrics"]["accuracy_answerable"], 1.0) + self.assertEqual(direct["metrics"]["coverage"], 1.0) + self.assertEqual(diffusion["metrics"]["coverage"], 0.5) + self.assertEqual(diffusion["metrics"]["selective_risk"], 0.0) + self.assertEqual(diffusion["abstained_rows"], 1) + self.assertIsNone(result["external_evaluation"]["mind2web"]["candidate_recall"]) + + def test_descriptive_stats_preserve_undefined_seed_metrics_as_null(self) -> None: + rows = [ + {"backend": "direct", "metrics": {"accuracy_answerable": 1.0}}, + {"backend": "direct", "metrics": {"accuracy_answerable": None}}, + ] + result = summary_tool.descriptive_seed_stats(rows) + metric = result["direct"]["accuracy_answerable"] + self.assertEqual(metric["mean"], 1.0) + self.assertEqual(metric["defined_seed_count"], 1) + self.assertEqual(metric["null_seed_count"], 1) + self.assertEqual(metric["seed_count"], 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 0000000000000000000000000000000000000000..a060202463fc43adfac7f68a7af7bab1a542b2ce --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,127 @@ +import math +import unittest + +from vons.contract import ( + DecisionResponse, + OptionProbability, + QuestionAnswer, + ResponseStatus, + derive_kasi_action, +) +from vons.kasi import KASIAdapter, KASICall, KASIProposal +from vons.models import ddim_step, linear_beta_schedule, require_torch + +try: + torch, _ = require_torch() +except RuntimeError: + torch = None + + +class KASIRegressionTests(unittest.TestCase): + def test_high_risk_proposal_derives_confirm_in_host_adapter(self) -> None: + adapter = KASIAdapter(tool_policy={"unlock": "high"}) + proposal = KASIProposal( + calls=(KASICall(name="unlock", arguments={"target": "lab"}),), + confidence=0.99, + risk="high", + ) + + decision = adapter.decide(proposal) + + self.assertEqual(decision.action.value, "confirm") + self.assertEqual(decision.reason, "unconfirmed_or_unknown_risk") + self.assertEqual(decision.calls, proposal.calls) + + def test_host_risk_cannot_be_lowered_by_model_risk(self) -> None: + adapter = KASIAdapter(tool_policy={"unlock": "high"}) + proposal = KASIProposal( + calls=(KASICall(name="unlock", arguments={"target": "lab"}),), + confidence=0.99, + risk="low", + ) + self.assertEqual(adapter.decide(proposal).action.value, "confirm") + + def test_exact_call_consent_is_required_for_arguments(self) -> None: + adapter = KASIAdapter(tool_policy={"unlock": "high"}) + first = KASICall(name="unlock", arguments={"target": "lab"}) + second = KASICall(name="unlock", arguments={"target": "vault"}) + proposal = KASIProposal(calls=(second,), confidence=0.99, risk="high") + self.assertEqual(adapter.decide(proposal, confirmed_calls={first.fingerprint()}).action.value, "confirm") + self.assertEqual(adapter.decide(proposal, confirmed_calls={second.fingerprint()}).action.value, "call") + self.assertEqual(adapter.decide(proposal, confirmed_tools={"unlock"}).action.value, "confirm") + + def test_unregistered_call_is_refused(self) -> None: + adapter = KASIAdapter() + proposal = KASIProposal(calls=(KASICall(name="read", arguments={}),), confidence=0.99, risk="low") + decision = adapter.decide(proposal) + self.assertEqual(decision.action.value, "refuse") + + def test_invalid_confidence_fails_closed(self) -> None: + self.assertEqual( + derive_kasi_action(proposed_calls=({"name": "read"},), confidence=math.nan, risk="low", tool_policy={"read": "low"}).value, + "clarify", + ) + + def test_malformed_tool_name_rejected(self) -> None: + with self.assertRaises(ValueError): + KASICall(name="unlock\nnow", arguments={}) + + def test_question_answer_invariants_and_response_parser(self) -> None: + with self.assertRaises(ValueError): + QuestionAnswer("q", "a", (OptionProbability("a", 0.5), OptionProbability("b", 0.5)), 0.9, ResponseStatus.ABSTAIN, "missing") + with self.assertRaises(ValueError): + QuestionAnswer("q", "c", (OptionProbability("a", 1.0),), 0.9, ResponseStatus.OK) + response = DecisionResponse.from_mapping( + { + "answers": [ + { + "question_id": "q", + "choice": "a", + "probabilities": [{"option": "a", "probability": 1.0}], + "confidence": 0.9, + "status": "ok", + } + ], + "backend": "direct", + "model_id": "test", + } + ) + self.assertEqual(response.to_mapping()["answers"][0]["choice"], "a") + + def test_low_confidence_derives_clarify(self) -> None: + adapter = KASIAdapter(tool_policy={"read": "low"}) + proposal = KASIProposal( + calls=(KASICall(name="read", arguments={"path": "state.json"}),), + confidence=0.2, + risk="low", + ) + + decision = adapter.decide(proposal) + + self.assertEqual(decision.action.value, "clarify") + self.assertEqual(decision.reason, "low_confidence") + + +@unittest.skipIf(torch is None, "optional torch dependency is unavailable") +class DiffusionRegressionTests(unittest.TestCase): + def test_linear_beta_schedule_is_deterministic_and_increasing(self) -> None: + first = linear_beta_schedule(8) + second = linear_beta_schedule(8) + + self.assertTrue(torch.equal(first, second)) + self.assertTrue(bool(torch.all(first[1:] > first[:-1]).item())) + + def test_ddim_step_preserves_score_vector_shape(self) -> None: + torch.manual_seed(17) + sample = torch.randn(2, 4) + predicted_noise = torch.randn(2, 4) + alpha = torch.tensor(0.8) + alpha_previous = torch.tensor(0.9) + + updated = ddim_step(sample, predicted_noise, alpha, alpha_previous) + + self.assertEqual(updated.shape, sample.shape) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_shape_benchmark.py b/tests/test_shape_benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..a5b5a53123ddb447af52f68f97299bff1a9e2b76 --- /dev/null +++ b/tests/test_shape_benchmark.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +from vons.data import Example + +_SPEC = importlib.util.spec_from_file_location("benchmark_onnx_shapes", Path(__file__).parents[1] / "tools/benchmark_onnx_shapes.py") +assert _SPEC and _SPEC.loader +_MODULE = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _MODULE +_SPEC.loader.exec_module(_MODULE) +_options_for = _MODULE._options_for +shape_cells = _MODULE.shape_cells + + +def _row(options: tuple[str, ...]) -> Example: + return Example( + id="fixture", + task_group="shape", + state="ready", + question="Choose", + options=options, + label=options[0], + answerable=True, + split="smoke", + provenance={}, + metadata={}, + ) + + +def test_shape_cells_keep_the_four_padding_comparisons_explicit() -> None: + cells = shape_cells(live_candidates=4, short_sequence=64, full_sequence=512) + assert [cell.cell_id for cell in cells] == [ + "direct_live_short", + "direct_padded_short", + "direct_padded_full", + "diffusion_padded_full", + ] + assert [(cell.backend, cell.slots, cell.sequence_length) for cell in cells] == [ + ("direct", 4, 64), + ("direct", 32, 64), + ("direct", 32, 512), + ("diffusion", 32, 512), + ] + + +def test_shape_fixture_expands_candidates_without_changing_live_prefix() -> None: + options = _options_for(_row(("a", "b")), 4) + assert options[:2] == ("a", "b") + assert len(options) == 4 + assert len(set(options)) == 4 + + +@pytest.mark.parametrize( + "kwargs", + [ + {"live_candidates": 1, "short_sequence": 64, "full_sequence": 512}, + {"live_candidates": 33, "short_sequence": 64, "full_sequence": 512}, + {"live_candidates": 4, "short_sequence": 512, "full_sequence": 64}, + ], +) +def test_shape_cells_reject_invalid_dimensions(kwargs: dict[str, int]) -> None: + with pytest.raises(ValueError): + shape_cells(**kwargs) diff --git a/tests/test_shared_bundle.py b/tests/test_shared_bundle.py new file mode 100644 index 0000000000000000000000000000000000000000..3c24d7c6d9a2fe71612f5a58d2617497870bb25d --- /dev/null +++ b/tests/test_shared_bundle.py @@ -0,0 +1,143 @@ +"""Offline graph-partition and release-integrity tests using small toy graphs.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + +onnx = pytest.importorskip("onnx") +np = pytest.importorskip("numpy") +spec = importlib.util.spec_from_file_location( + "vons_shared_export", Path(__file__).parents[1] / "tools/export_shared_bundle.py", +) +assert spec is not None and spec.loader is not None +exporter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(exporter) +digest = exporter.digest +export_bundle = exporter.export_bundle +partition_graphs = exporter.partition_graphs +verify_manifest = exporter.verify_manifest + + +def toy_model(backend: str, *, weight: float = 1.0): + helper = onnx.helper + tensor = onnx.TensorProto + inputs = [helper.make_tensor_value_info(name, tensor.INT64, ["batch", "options", 2]) + for name in ("input_ids", "attention_mask", "token_type_ids")] + inputs.append(helper.make_tensor_value_info("option_mask", tensor.BOOL, ["batch", "options"])) + values = [helper.make_tensor_value_info("view_19", tensor.FLOAT, ["batch", "options", 2]), + helper.make_tensor_value_info("div", tensor.FLOAT, ["batch", 2])] + weights = [onnx.numpy_helper.from_array(np.eye(2, dtype=np.float32) * weight, + name="encoder.weight"), + onnx.numpy_helper.from_array(np.array([1], np.int64), name="axis1"), + onnx.numpy_helper.from_array(np.array([2], np.int64), name="axis2")] + nodes = [helper.make_node("Cast", ["input_ids"], ["cast"], to=tensor.FLOAT), + helper.make_node("MatMul", ["cast", "encoder.weight"], ["view_19"]), + helper.make_node("ReduceMean", ["view_19", "axis1"], ["div"], keepdims=0), + helper.make_node("ReduceSum", ["div", "axis1"], ["answerability"], keepdims=0)] + output = "logits" if backend == "direct" else "scores" + if backend == "direct": + nodes.append(helper.make_node("ReduceSum", ["view_19", "axis2"], [output], keepdims=0)) + else: + inputs.append(helper.make_tensor_value_info("initial_noise", tensor.FLOAT, ["batch", "options"])) + nodes.extend([ + helper.make_node("Shape", ["token_type_ids"], ["batch_shape"], start=0, end=1), + helper.make_node("Expand", ["answerability", "batch_shape"], ["expanded"]), + helper.make_node("Unsqueeze", ["expanded", "axis1"], ["scalar"]), + helper.make_node("Add", ["initial_noise", "scalar"], [output]), + ]) + outputs = [helper.make_tensor_value_info(output, tensor.FLOAT, ["batch", "options"]), + helper.make_tensor_value_info("answerability", tensor.FLOAT, ["batch"])] + model = helper.make_model(helper.make_graph(nodes, "toy", inputs, outputs, weights, + value_info=values), + opset_imports=[helper.make_opsetid("", 18)]) + model.ir_version = 10 + return model + + +def test_partition_keeps_outputs_and_removes_token_dependency(): + parts = partition_graphs(toy_model("direct"), toy_model("diffusion")) + for model in parts.values(): + onnx.checker.check_model(model) + assert [x.name for x in parts["encoder"].graph.output] == ["candidate_embeddings", "pooled"] + assert all("token_type_ids" not in n.input for n in parts["diffusion"].graph.node) + + +def test_different_encoders_cannot_be_silently_shared(): + with pytest.raises(ValueError, match="parameter tensors differ"): + partition_graphs(toy_model("direct"), toy_model("diffusion", weight=2)) + + +def test_partition_matches_full_graph_with_nonzero_noise(): + ort = pytest.importorskip("onnxruntime") + original = {name: toy_model(name) for name in ("direct", "diffusion")} + parts = partition_graphs(original["direct"], original["diffusion"]) + options = ort.SessionOptions() + options.intra_op_num_threads = 1 + sessions = {key: ort.InferenceSession(value.SerializeToString(), options, + providers=["CPUExecutionProvider"]) + for key, value in parts.items()} + feed = {name: np.array([[[1, 2], [3, 4], [5, 6]]], np.int64) + for name in ("input_ids", "attention_mask", "token_type_ids")} + feed["option_mask"] = np.array([[True, True, True]]) + encoded = sessions["encoder"].run(None, feed) + for backend, model in original.items(): + shared = {"candidate_embeddings": encoded[0], "pooled": encoded[1], + "option_mask": feed["option_mask"]} + full_feed = dict(feed) + if backend == "diffusion": + shared.pop("candidate_embeddings") + shared["initial_noise"] = np.array([[0.2, -0.7, 1.1]], np.float32) + full_feed["initial_noise"] = shared["initial_noise"] + full = ort.InferenceSession(model.SerializeToString(), options, + providers=["CPUExecutionProvider"]).run(None, full_feed) + split = sessions[backend].run(None, shared) + for expected, actual in zip(full, split, strict=True): + np.testing.assert_allclose(actual, expected, atol=1e-6) + + +def make_bundle(tmp_path: Path): + for backend in ("direct", "diffusion"): + onnx.save(toy_model(backend), tmp_path / f"{backend}.onnx") + tokenizer = tmp_path / "tokenizer" + tokenizer.mkdir() + (tokenizer / "tokenizer.json").write_text("{}") + (tokenizer / "tokenizer_config.json").write_text("{}") + target = tmp_path / "bundle" + manifest = export_bundle(tmp_path / "direct.onnx", tmp_path / "diffusion.onnx", tokenizer, target) + return target, manifest + + +def test_asset_count_includes_config_calibration_and_both_heads(tmp_path): + target, manifest = make_bundle(tmp_path) + assert {r["role"] for r in manifest["files"]} >= { + "encoder_graph", "direct_graph", "diffusion_graph", "calibration", "config", "tokenizer", + } + assert manifest["model_asset_bytes"] == sum((target / r["path"]).stat().st_size + for r in manifest["files"]) + assert verify_manifest(target / "manifest.json")["pass"] + with pytest.raises(ValueError, match="new or empty"): + export_bundle(tmp_path / "direct.onnx", tmp_path / "diffusion.onnx", + tmp_path / "tokenizer", target) + + +def test_tamper_and_external_expected_digest_are_rejected(tmp_path): + target, _ = make_bundle(tmp_path) + path = target / "manifest.json" + with pytest.raises(ValueError, match="release manifest digest"): + verify_manifest(path, expected_digest="0" * 64) + (target / "config.json").write_text("tampered") + with pytest.raises(ValueError, match="asset digest"): + verify_manifest(path) + + +def test_referenced_assets_cannot_be_omitted(tmp_path): + target, manifest = make_bundle(tmp_path) + manifest["files"] = [r for r in manifest["files"] if r["path"] != "calibration.json"] + manifest["model_asset_bytes"] = sum(r["bytes"] for r in manifest["files"]) + path = target / "manifest.json" + path.write_text(json.dumps(manifest)) + path.with_suffix(".sha256").write_text(digest(path)) + with pytest.raises(ValueError, match="omits required"): + verify_manifest(path) diff --git a/tools/archive_manifest.py b/tools/archive_manifest.py new file mode 100644 index 0000000000000000000000000000000000000000..6094c6d0dd7765919e10d77990cfbe492a5b892c --- /dev/null +++ b/tools/archive_manifest.py @@ -0,0 +1,33 @@ +"""Archive exact manifest bytes for a browser measurement provenance record.""" + +from __future__ import annotations + +import argparse +import hashlib +from pathlib import Path + + +def archive(source: Path, output: Path, expected_sha256: str) -> dict[str, object]: + data = source.read_bytes() + actual_sha256 = hashlib.sha256(data).hexdigest() + if actual_sha256 != expected_sha256: + raise ValueError(f"source manifest hash {actual_sha256} != expected {expected_sha256}") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(data) + if output.read_bytes() != data: + raise OSError(f"archived manifest bytes do not match source: {output}") + return {"source": str(source), "output": str(output), "bytes": len(data), "sha256": actual_sha256} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--expected-sha256", required=True) + args = parser.parse_args() + result = archive(args.source, args.output, args.expected_sha256) + print(result) + + +if __name__ == "__main__": + main() diff --git a/tools/audit_local_model_alignment.py b/tools/audit_local_model_alignment.py new file mode 100644 index 0000000000000000000000000000000000000000..099ed060ec9e8a491f192a46d347fc589383654f --- /dev/null +++ b/tools/audit_local_model_alignment.py @@ -0,0 +1,89 @@ +"""Audit saved local-model suitability records without making new model calls.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Any + + +def sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def audit(path: Path) -> dict[str, Any]: + raw = path.read_bytes() + report = json.loads(raw) + records = report["records"] + groups: Counter[str] = Counter() + evidence = [] + correct = 0 + strict_shape = 0 + null_on_abstain = 0 + prompt_set = set() + for record in records: + prompt = json.loads(record["prompt"]) + parsed = json.loads(record["response"]) if record["response"] else None + valid = (isinstance(parsed, dict) and set(parsed) == {"choice", "abstain"} + and isinstance(parsed["abstain"], bool) + and (parsed["choice"] is None or parsed["choice"] in prompt["candidates"])) + strict_shape += valid + if valid: + correct += ((record["expected_answerable"] and not parsed["abstain"] + and parsed["choice"] == record["expected_label"]) + or (not record["expected_answerable"] and parsed["abstain"])) + null_on_abstain += not parsed["abstain"] or parsed["choice"] is None + key = json.dumps({"expected_label": record["expected_label"], + "answerable": record["expected_answerable"], "parsed": parsed}, + sort_keys=True) + groups[key] += 1 + prompt_hash = sha256(record["prompt"].encode()) + prompt_set.add(prompt_hash) + evidence.append({"example_id": record["example_id"], "prompt_sha256": prompt_hash, + "response_sha256": sha256((record["response"] or "").encode()), + "accepted_for_training": False, "reason": "prompt_label_alignment_unresolved"}) + return {"model": report["model"], "role_metadata": report["role"], + "raw_path": str(path), "raw_sha256": sha256(raw), "rows": len(records), + "known_label_accuracy": correct / len(records) if records else None, + "strict_json_shape_and_candidate_rows": strict_shape, + "null_on_abstention_contract_rows": null_on_abstain, + "unique_prompt_count": len(prompt_set), "prompt_hashes": sorted(prompt_set), + "response_groups": [{**json.loads(key), "rows": count} for key, count in sorted(groups.items())], + "record_hashes": evidence, + "digest": records[0]["metadata"].get("digest") if records else None, + "upstream_revision": None, + "upstream_revision_reason": "not established by saved local metadata"} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, action="append", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists(): + parser.error("refusing to overwrite historical audit") + models = [audit(path) for path in args.input] + identical = all(row["prompt_hashes"] == models[0]["prompt_hashes"] for row in models) + result = {"schema": "vons.local-model-alignment/v1", "models": models, + "identical_prompt_sets_across_roles": identical, + "source_sha256": sha256(Path(__file__).read_bytes()), + "new_model_calls": 0, "teacher_outputs_used_for_training": False, + "interpretation": [ + "Every answerable pilot label is call_tool, while all three models select respond_directly.", + "The saved state does not identify a required tool or a rule distinguishing those two actions. This is a prompt/label specification limitation, not a proven model error rate on real workflows.", + "Clarify is an available action for underspecified inputs, but the label instead requires abstention; the prompt does not state that mapping.", + "Role names are metadata; the benchmark uses identical task prompts and one generic system instruction, not distinct writer/judge/analyst evaluations.", + "Muse's 0.09 comes from its 18 abstention booleans. Its nonnull choice on those rows would violate Vons's public abstention contract despite passing the benchmark JSON schema.", + "Historical scores are preserved. No general model ranking, swapped-role validation, development-task benchmark or teacher-training admission follows from this audit.", + ]} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") + print(json.dumps({"models": len(models), "identical_prompt_sets_across_roles": identical, + "rows": sum(row["rows"] for row in models)})) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmark_onnx_bundle.py b/tools/benchmark_onnx_bundle.py new file mode 100644 index 0000000000000000000000000000000000000000..f41e8628e1d6272fb8c51deaf1563ddaf22253e2 --- /dev/null +++ b/tools/benchmark_onnx_bundle.py @@ -0,0 +1,95 @@ +"""Measure cold load and end-to-end warm latency for an exported bundle.""" + +from __future__ import annotations + +import argparse +import json +import platform +import time +from pathlib import Path + +import numpy as np +import onnxruntime as ort +from transformers import AutoTokenizer + +from vons.data import read_jsonl + + +def percentile(values: list[float], fraction: float) -> float: + ordered = sorted(values) + index = (len(ordered) - 1) * fraction + lower, upper = int(index), min(int(index) + 1, len(ordered) - 1) + return ordered[lower] + (ordered[upper] - ordered[lower]) * (index - lower) + + +def benchmark(bundle_manifest: Path, input_path: Path, limit: int) -> dict[str, object]: + manifest = json.loads(bundle_manifest.read_text(encoding="utf-8")) + tokenizer = AutoTokenizer.from_pretrained(manifest["tokenizer"]) + rows = read_jsonl(input_path)[:limit] + load_started = time.perf_counter() + session = ort.InferenceSession(manifest["onnx"], providers=["CPUExecutionProvider"]) + cold_load_ms = (time.perf_counter() - load_started) * 1000 + backend = manifest["backend"] + option_count = int(manifest["option_count"]) + sequence_length = int(manifest["sequence_length"]) + rng = np.random.default_rng(7) + latencies: list[float] = [] + for row in rows: + texts = [f"{row.state}\nQuestion: {row.question}\nCandidate: {option}" for option in row.options] + started = time.perf_counter() + encoded = tokenizer(texts, padding="max_length", truncation=True, max_length=sequence_length, return_tensors="np") + if backend == "diffusion": + pad = option_count - len(row.options) + if pad < 0: + raise ValueError("row has more candidates than the fixed diffusion bundle") + feed = { + "input_ids": np.pad(encoded["input_ids"][None, :, :], ((0, 0), (0, pad), (0, 0))), + "attention_mask": np.pad(encoded["attention_mask"][None, :, :], ((0, 0), (0, pad), (0, 0))), + "token_type_ids": np.pad(encoded["token_type_ids"][None, :, :], ((0, 0), (0, pad), (0, 0))), + "option_mask": np.asarray([[True] * len(row.options) + [False] * pad], dtype=np.bool_), + "initial_noise": rng.standard_normal((1, option_count), dtype=np.float32), + } + else: + feed = { + "input_ids": encoded["input_ids"][None, :, :], + "attention_mask": encoded["attention_mask"][None, :, :], + "token_type_ids": encoded["token_type_ids"][None, :, :], + "option_mask": np.ones((1, len(row.options)), dtype=np.bool_), + } + session.run(None, feed) + latencies.append((time.perf_counter() - started) * 1000) + return { + "backend": backend, + "bundle": manifest["onnx"], + "input": str(input_path), + "rows": len(latencies), + "provider": "CPUExecutionProvider", + "onnxruntime_version": ort.__version__, + "platform": platform.platform(), + "candidate_slots": option_count if backend == "diffusion" else "row-defined", + "sequence_length": sequence_length, + "padding": "max_length", + "inference_steps": manifest.get("inference_steps") if backend == "diffusion" else None, + "cold_load_ms": cold_load_ms, + "warm_p50_ms": percentile(latencies, 0.5) if latencies else None, + "warm_p95_ms": percentile(latencies, 0.95) if latencies else None, + "latency_scope": "tokenization plus ONNX Runtime inference", + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--limit", type=int, default=20) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + result = benchmark(args.manifest, args.input, args.limit) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/benchmark_onnx_shapes.py b/tools/benchmark_onnx_shapes.py new file mode 100644 index 0000000000000000000000000000000000000000..12d9c9da26642dc76d3faac5709cd8293abadbdd --- /dev/null +++ b/tools/benchmark_onnx_shapes.py @@ -0,0 +1,346 @@ +"""Measure the four-cell ONNX shape/padding decomposition. + +This is a CPU reference runner for G3. It intentionally keeps the historical +20-row benchmark untouched and writes raw samples for the matched Direct and +Diffusion shape cells described in ``docs/BENCHMARK_PROTOCOL.md``. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import onnxruntime as ort + +from vons.data import Example, read_jsonl + + +@dataclass(frozen=True) +class ShapeCell: + cell_id: str + backend: str + slots: int + sequence_length: int + live_candidates: int + + +def shape_cells(*, live_candidates: int, short_sequence: int, full_sequence: int, slots: int = 32) -> tuple[ShapeCell, ...]: + if live_candidates < 2 or slots < live_candidates: + raise ValueError("live_candidates must be at least 2 and no larger than slots") + if short_sequence < 1 or full_sequence < short_sequence: + raise ValueError("sequence buckets must be positive and ordered") + return ( + ShapeCell("direct_live_short", "direct", live_candidates, short_sequence, live_candidates), + ShapeCell("direct_padded_short", "direct", slots, short_sequence, live_candidates), + ShapeCell("direct_padded_full", "direct", slots, full_sequence, live_candidates), + ShapeCell("diffusion_padded_full", "diffusion", slots, full_sequence, live_candidates), + ) + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _canonical_bytes(value: Any) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _runtime_manifest(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("schema_version") != "vons.bundle.manifest/v1": + return payload + root = path.parent + graph = next(item for item in payload["files"] if item["role"] == "model_graph") + tokenizer_json = next(item for item in payload["files"] if item["role"] == "tokenizer" and item["path"].endswith("tokenizer.json")) + metadata = payload["metadata"] + return { + "backend": metadata["backend"], + "onnx": str(root / graph["path"]), + "tokenizer": str(root / Path(tokenizer_json["path"]).parent), + "option_count": metadata.get("option_count", 32), + "sequence_length": metadata.get("sequence_length", 512), + "bundle_manifest": str(path), + } + + +def _options_for(row: Example, live_candidates: int) -> tuple[str, ...]: + if len(row.options) >= live_candidates: + return row.options[:live_candidates] + values = list(row.options) + base = values[0] + for index in range(len(values), live_candidates): + values.append(f"{base}__shape_fixture_{index}") + return tuple(values) + + +def _noise(seed: int, case_index: int, slots: int) -> np.ndarray: + generator = np.random.default_rng(seed + case_index) + return generator.standard_normal((1, slots), dtype=np.float32) + + +def _token_arrays(tokenizer: Any, row: Example, options: tuple[str, ...], sequence_length: int) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + ids = np.zeros((len(options), sequence_length), dtype=np.int64) + attention = np.zeros_like(ids) + token_types = np.zeros_like(ids) + padding_id = tokenizer.token_to_id("[PAD]") + if padding_id is None: + raise ValueError("bundle tokenizer has no [PAD] token") + ids.fill(int(padding_id)) + for index, option in enumerate(options): + text = f"{row.state}\nQuestion: {row.question}\nCandidate: {option}" + encoded = tokenizer.encode(text) + if len(encoded.ids) > sequence_length: + raise ValueError( + f"case {row.id} needs {len(encoded.ids)} tokens but the cell allows {sequence_length}" + ) + ids[index, : len(encoded.ids)] = encoded.ids + attention[index, : len(encoded.attention_mask)] = encoded.attention_mask + token_types[index, : len(encoded.type_ids)] = encoded.type_ids + return ids, attention, token_types + + +def _feed( + token_arrays: tuple[np.ndarray, np.ndarray, np.ndarray], + options: tuple[str, ...], + cell: ShapeCell, + noise: np.ndarray, + padding_id: int, +) -> tuple[dict[str, np.ndarray], dict[str, int | str]]: + token_ids, attention, token_types = token_arrays + ids = np.zeros((1, cell.slots, cell.sequence_length), dtype=np.int64) + masks = np.zeros_like(ids) + types = np.zeros_like(ids) + ids.fill(int(padding_id)) + ids[0, : len(options)] = token_ids + masks[0, : len(options)] = attention + types[0, : len(options)] = token_types + option_mask = np.asarray([[True] * len(options) + [False] * (cell.slots - len(options))], dtype=np.bool_) + feed: dict[str, np.ndarray] = { + "input_ids": ids, + "attention_mask": masks, + "token_type_ids": types, + "option_mask": option_mask, + } + if cell.backend == "diffusion": + feed["initial_noise"] = noise + shape_info: dict[str, int | str] = { + "live_candidates": len(options), + "allocated_candidates": cell.slots, + "live_tokens": int(np.sum(attention)), + "sequence_length": cell.sequence_length, + } + return feed, shape_info + + +def _postprocess(outputs: list[np.ndarray], backend: str, live_candidates: int) -> None: + scores = outputs[0][0, :live_candidates] + if np.any(~np.isfinite(scores)): + raise ValueError(f"{backend} returned non-finite live scores") + shifted = scores - np.max(scores) + probabilities = np.exp(shifted) + probabilities /= np.sum(probabilities) + answerability = float(1.0 / (1.0 + np.exp(-float(outputs[1][0])))) + if not np.all(np.isfinite(probabilities)) or not np.isfinite(answerability): + raise ValueError(f"{backend} returned non-finite postprocessed values") + + +def _percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + + +def _cell_summary(samples: list[dict[str, Any]]) -> dict[str, Any]: + fields = ("tokenize_ms", "prepare_feed_ms", "inference_and_readback_ms", "postprocess_ms", "total_request_ms") + return { + "samples": len(samples), + **{ + field: {"p50": _percentile([float(item[field]) for item in samples], 0.5), "p95": _percentile([float(item[field]) for item in samples], 0.95)} + for field in fields + }, + "failures": sum(1 for item in samples if item["status"] != "ok"), + } + + +def benchmark( + direct_manifest: Path, + diffusion_manifest: Path, + input_path: Path, + *, + limit: int = 4, + session_starts: int = 1, + warmup: int = 2, + repeats: int = 5, + seed: int = 7, + live_candidates: int = 4, + short_sequence: int = 64, + full_sequence: int = 512, +) -> dict[str, Any]: + if limit < 1 or session_starts < 1 or warmup < 0 or repeats < 1: + raise ValueError("limit/session_starts/repeats must be positive and warmup cannot be negative") + cells = shape_cells(live_candidates=live_candidates, short_sequence=short_sequence, full_sequence=full_sequence) + manifests = {"direct": _runtime_manifest(direct_manifest), "diffusion": _runtime_manifest(diffusion_manifest)} + rows = read_jsonl(input_path)[:limit] + if not rows: + raise ValueError("input contains no rows") + tokenizers: dict[str, Any] = {} + tokenizer_hashes = { + backend: _sha256_bytes((Path(manifest["tokenizer"]) / "tokenizer.json").read_bytes()) + for backend, manifest in manifests.items() + } + bundle_hashes = { + backend: _sha256_bytes(Path(manifest["onnx"]).read_bytes()) + for backend, manifest in manifests.items() + } + raw_samples: list[dict[str, Any]] = [] + for backend, manifest in manifests.items(): + from tokenizers import Tokenizer + + tokenizers[backend] = Tokenizer.from_file(str(Path(manifest["tokenizer"]) / "tokenizer.json")) + for session_index in range(session_starts): + sessions: dict[str, Any] = {} + for backend in ("direct", "diffusion"): + sessions[backend] = ort.InferenceSession(manifests[backend]["onnx"], providers=["CPUExecutionProvider"]) + for cell in cells: + tokenizer = tokenizers[cell.backend] + padding_id = tokenizer.token_to_id("[PAD]") + if padding_id is None: + raise ValueError("bundle tokenizer has no [PAD] token") + noise_by_case = { + index: _noise(seed, index, cell.slots) for index in range(len(rows)) + } + for row_index, row in enumerate(rows): + options = _options_for(row, live_candidates) + noise = noise_by_case[row_index] + input_identity = _canonical_bytes({"state": row.state, "question": row.question, "options": options}) + text_hash = _sha256_bytes(input_identity) + noise_hash = _sha256_bytes(noise.tobytes()) if cell.backend == "diffusion" else None + for warmup_index in range(warmup): + token_arrays = _token_arrays(tokenizer, row, options, cell.sequence_length) + feed, _ = _feed(token_arrays, options, cell, noise, int(padding_id)) + sessions[cell.backend].run(None, feed) + for repeat_index in range(repeats): + started = time.perf_counter() + tokenize_started = time.perf_counter() + # Tokenization is kept separate from feed packing to expose sequence padding cost. + tokenized = _token_arrays(tokenizer, row, options, cell.sequence_length) + tokenize_ms = (time.perf_counter() - tokenize_started) * 1000 + prepare_started = time.perf_counter() + feed, shape_info = _feed(tokenized, options, cell, noise, int(padding_id)) + prepare_feed_ms = (time.perf_counter() - prepare_started) * 1000 + inference_started = time.perf_counter() + output_values = sessions[cell.backend].run(None, feed) + inference_ms = (time.perf_counter() - inference_started) * 1000 + postprocess_started = time.perf_counter() + _postprocess(output_values, cell.backend, len(options)) + postprocess_ms = (time.perf_counter() - postprocess_started) * 1000 + total_ms = (time.perf_counter() - started) * 1000 + raw_samples.append( + { + "run_id": f"cpu-shape-{session_index}", + "session_start": session_index, + "case_id": row.id, + "repeat": repeat_index, + "warmup_excluded": warmup, + "cell": cell.cell_id, + "backend": cell.backend, + "requested_provider": "CPUExecutionProvider", + "observed_provider_status": "cpu_ort", + "bundle_hash": bundle_hashes[cell.backend], + "tokenizer_hash": tokenizer_hashes[cell.backend], + "input_hash": text_hash, + "noise_hash": noise_hash, + "platform": platform.platform(), + **shape_info, + "tokenize_ms": tokenize_ms, + "prepare_feed_ms": prepare_feed_ms, + "inference_and_readback_ms": inference_ms, + "postprocess_ms": postprocess_ms, + "total_request_ms": total_ms, + "status": "ok", + "error_kind": None, + } + ) + del sessions + summaries = { + cell.cell_id: _cell_summary([item for item in raw_samples if item["cell"] == cell.cell_id]) + for cell in cells + } + comparisons = { + "direct_padding_candidates_ms": _difference(summaries, "direct_padded_short", "direct_live_short", "inference_and_readback_ms"), + "direct_padding_sequence_ms": _difference(summaries, "direct_padded_full", "direct_padded_short", "inference_and_readback_ms"), + "matched_diffusion_minus_direct_ms": _difference(summaries, "diffusion_padded_full", "direct_padded_full", "inference_and_readback_ms"), + } + return { + "schema_version": "vons.shape-benchmark/v1", + "method": "CPU ONNX Runtime shape decomposition; same live texts and masks across paired cells", + "input": str(input_path), + "rows": len(rows), + "session_starts": session_starts, + "warmup": warmup, + "repeats": repeats, + "seed": seed, + "cells": [cell.__dict__ for cell in cells], + "onnxruntime_version": ort.__version__, + "platform": platform.platform(), + "summaries": summaries, + "comparisons": comparisons, + "raw_samples": raw_samples, + } + + +def _difference(summaries: dict[str, Any], right: str, left: str, field: str) -> float | None: + right_value = summaries[right][field]["p50"] + left_value = summaries[left][field]["p50"] + if right_value is None or left_value is None: + return None + return float(right_value - left_value) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--direct-manifest", required=True, type=Path) + parser.add_argument("--diffusion-manifest", required=True, type=Path) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--limit", type=int, default=4) + parser.add_argument("--session-starts", type=int, default=1) + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--seed", type=int, default=7) + parser.add_argument("--live-candidates", type=int, default=4) + parser.add_argument("--short-sequence", type=int, default=64) + parser.add_argument("--full-sequence", type=int, default=512) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + result = benchmark( + args.direct_manifest, + args.diffusion_manifest, + args.input, + limit=args.limit, + session_starts=args.session_starts, + warmup=args.warmup, + repeats=args.repeats, + seed=args.seed, + live_candidates=args.live_candidates, + short_sequence=args.short_sequence, + full_sequence=args.full_sequence, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"schema_version": result["schema_version"], "rows": result["rows"], "summaries": result["summaries"], "comparisons": result["comparisons"]}, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_claim_ledger.py b/tools/build_claim_ledger.py new file mode 100644 index 0000000000000000000000000000000000000000..5fa0172988cba15b1bb5db9cb727a5042e02cdf8 --- /dev/null +++ b/tools/build_claim_ledger.py @@ -0,0 +1,53 @@ +"""Create a field-level claim ledger from an explicit frozen JSON specification.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +from pathlib import Path +from typing import Any + + +def resolve(document: Any, pointer: str) -> Any: + for part in pointer.split("/")[1:]: + key = part.replace("~1", "/").replace("~0", "~") + document = document[int(key)] if isinstance(document, list) else document[key] + return document + + +def build(spec: Path, output: Path) -> int: + root = Path(__file__).resolve().parents[1] + records = [] + for item in json.loads(spec.read_text())["claims"]: + source = root / item["source_path"] + raw = source.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest != item["source_sha256"]: + raise ValueError(f"claim source changed: {item['source_path']}") + value = resolve(json.loads(raw), item["json_pointer"]) + operation = item.get("operation", "identity") + if operation == "length": + value = len(value) + elif operation != "identity": + raise ValueError("unsupported ledger operation") + records.append({**item, "operation": operation, + "value_json": json.dumps(value, allow_nan=False, ensure_ascii=False)}) + if len({item["claim_id"] for item in records}) != len(records): + raise ValueError("claim ids must be unique") + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("x", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["claim_id", "source_path", "source_sha256", + "json_pointer", "operation", "value_json", "scope"]) + writer.writeheader() + writer.writerows(records) + return len(records) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spec", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print(json.dumps({"claims": build(args.spec, args.output), "output": str(args.output)})) diff --git a/tools/build_mind2web_manifest.py b/tools/build_mind2web_manifest.py new file mode 100644 index 0000000000000000000000000000000000000000..4706087e5b3c9357bb375c7ad8b642f67f8282bf --- /dev/null +++ b/tools/build_mind2web_manifest.py @@ -0,0 +1,95 @@ +"""Build a non-public-path provenance manifest for an approved Mind2Web split.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections.abc import Sequence +from pathlib import Path +from typing import Any + + +def _file_digest(path: Path) -> dict[str, int | str]: + digest = hashlib.sha256() + byte_count = 0 + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + byte_count += len(chunk) + digest.update(chunk) + return {"sha256": digest.hexdigest(), "bytes": byte_count} + + +def build_manifest( + *, + variant: str, + source_revision: str, + permitted_use: str, + approval_record: str, + preprocessing_revision: str, + retriever_revision: str, + splits: Sequence[tuple[str, Path]], +) -> dict[str, Any]: + if not variant or not source_revision or not approval_record: + raise ValueError("variant, source_revision and approval_record are required") + if permitted_use not in {"evaluation_only", "evaluation_and_training"}: + raise ValueError("permitted_use must be evaluation_only or evaluation_and_training") + if not splits: + raise ValueError("at least one split is required") + names = [name for name, _ in splits] + if any(not name for name in names) or len(names) != len(set(names)): + raise ValueError("split names must be non-empty and unique") + + split_records: list[dict[str, int | str]] = [] + for name, path in splits: + if not path.is_file(): + raise FileNotFoundError(path) + split_records.append({"name": name, **_file_digest(path)}) + return { + "schema": "vons.mind2web-provenance/v1", + "variant": variant, + "source_revision": source_revision, + "permitted_use": permitted_use, + "approval_record": approval_record, + "preprocessing_revision": preprocessing_revision, + "retriever_revision": retriever_revision, + "splits": split_records, + "raw_data_paths": "omitted from public artifacts", + "scope": "Mind2Web candidate recall and conditional selection only", + } + + +def _split_argument(value: str) -> tuple[str, Path]: + name, separator, path = value.partition("=") + if not separator or not name or not path: + raise argparse.ArgumentTypeError("split must be NAME=PATH") + return name, Path(path) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--variant", required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--permitted-use", required=True, choices=["evaluation_only", "evaluation_and_training"]) + parser.add_argument("--approval-record", required=True) + parser.add_argument("--preprocessing-revision", required=True) + parser.add_argument("--retriever-revision", required=True) + parser.add_argument("--split", action="append", type=_split_argument, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + manifest = build_manifest( + variant=args.variant, + source_revision=args.source_revision, + permitted_use=args.permitted_use, + approval_record=args.approval_record, + preprocessing_revision=args.preprocessing_revision, + retriever_revision=args.retriever_revision, + splits=args.split, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/tools/evaluate_mind2web.py b/tools/evaluate_mind2web.py new file mode 100644 index 0000000000000000000000000000000000000000..db4f38da4b1c53043a85fd23de37bac04174e1b2 --- /dev/null +++ b/tools/evaluate_mind2web.py @@ -0,0 +1,182 @@ +"""Run the restricted Mind2Web candidate/selection evaluation. + +The input rows must be normalized JSONL rows accepted by +``Mind2WebExample.from_mapping``. Predictions are intentionally a separate +JSONL file so retrieval misses and missing selections remain observable. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any, NamedTuple + +from vons.mind2web import Mind2WebExample, evaluate_mind2web, task_bootstrap_intervals + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + if not line.strip(): + continue + value = json.loads(line) + if not isinstance(value, Mapping): + raise TypeError(f"{path}:{line_number}: expected a JSON object") + rows.append(dict(value)) + return rows + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _string_sequence(value: Any, label: str) -> tuple[str, ...]: + if isinstance(value, (str, bytes, bytearray)) or not isinstance(value, Sequence): + raise TypeError(f"{label} must be a JSON list") + values: list[str] = [] + for item in value: + candidate = item.get("id") if isinstance(item, Mapping) else item + if not isinstance(candidate, str) or not candidate: + raise TypeError(f"{label} ids must be non-empty strings") + values.append(candidate) + if len(values) != len(set(values)): + raise ValueError(f"{label} ids must be unique") + return tuple(values) + + +class PredictionBundle(NamedTuple): + generated_candidates: dict[str, tuple[str, ...]] + selections: dict[str, str | None] + declared_k: int | None + + +def load_predictions(path: Path) -> PredictionBundle: + generated_candidates: dict[str, tuple[str, ...]] = {} + selections: dict[str, str | None] = {} + declared_ks: set[int] = set() + missing_k = False + for row in _read_jsonl(path): + example_id = row.get("id") + if not isinstance(example_id, str) or not example_id: + raise ValueError(f"{path}: prediction requires id") + key = example_id + if key in generated_candidates: + raise ValueError(f"{path}: duplicate prediction id {key}") + if "generated_candidates" not in row: + raise ValueError(f"{path}: prediction requires generated_candidates") + generated_candidates[key] = _string_sequence(row["generated_candidates"], "generated_candidates") + selection = row.get("selection", row.get("selected_id")) + selections[key] = None if selection in (None, "") else str(selection) + row_k = row.get("k") + if row_k is None: + missing_k = True + elif isinstance(row_k, bool) or not isinstance(row_k, int) or row_k <= 0: + raise ValueError(f"{path}: prediction k must be a positive integer") + else: + declared_ks.add(row_k) + if declared_ks and missing_k: + raise ValueError(f"{path}: every prediction row must declare k when any row does") + if len(declared_ks) > 1: + raise ValueError(f"{path}: prediction rows must use one k, found {sorted(declared_ks)}") + return PredictionBundle( + generated_candidates=generated_candidates, + selections=selections, + declared_k=next(iter(declared_ks), None), + ) + + +def evaluate_files( + rows_path: Path, + predictions_path: Path, + ks: Sequence[int], + split: str | None = None, +) -> dict[str, Any]: + raw_rows = _read_jsonl(rows_path) + if split is not None: + raw_rows = [row for row in raw_rows if row.get("split") == split] + examples = [Mind2WebExample.from_mapping(row) for row in raw_rows] + if len({example.example_id for example in examples}) != len(examples): + raise ValueError(f"{rows_path}: duplicate gold row ids") + predictions = load_predictions(predictions_path) + generated_candidates = predictions.generated_candidates + example_ids = {row.example_id for row in examples} + unknown_prediction_ids = sorted(set(generated_candidates) - example_ids) + if unknown_prediction_ids: + raise ValueError(f"predictions contain unknown example ids: {unknown_prediction_ids[:5]}") + if not ks or any(k <= 0 for k in ks): + raise ValueError("ks must contain positive integers") + if len(set(ks)) != len(ks): + raise ValueError("ks must not contain duplicates") + + if predictions.declared_k is not None and tuple(ks) != (predictions.declared_k,): + raise ValueError( + f"predictions are k={predictions.declared_k}; evaluate exactly that single k, not {list(ks)}" + ) + selection_is_k_specific = predictions.declared_k is not None or len(ks) == 1 + selections = predictions.selections if selection_is_k_specific else {} + metric_mappings: list[dict[str, Any]] = [] + bootstrap_by_k: dict[str, dict[str, Any]] = {} + for k in ks: + metric = evaluate_mind2web(examples, generated_candidates, selections, k=k) + mapping = metric.to_mapping() + if not selection_is_k_specific: + mapping["selection_accuracy_given_recall"] = None + mapping["complete_case_selection_accuracy_given_recall"] = None + mapping["selection_metrics_status"] = "not_evaluated_recall_only" + metric_mappings.append(mapping) + bootstrap = task_bootstrap_intervals(examples, generated_candidates, selections, k=k) + if not selection_is_k_specific: + bootstrap["selection_accuracy_given_recall_task_macro_ci95"] = None + bootstrap_by_k[str(k)] = bootstrap + return { + "schema": "vons.mind2web-evaluation/v1", + "rows_path": str(rows_path), + "predictions_path": str(predictions_path), + "rows": len(examples), + "prediction_rows": len(generated_candidates), + "split": split, + "input_sha256": { + "rows": _file_sha256(rows_path), + "predictions": _file_sha256(predictions_path), + }, + "prediction_k": predictions.declared_k, + "metrics_by_k": metric_mappings, + "task_bootstrap_by_k": bootstrap_by_k, + "selection_evaluation": { + "ranking_and_selection_predictions_supplied_once": True, + "selection_reused_across_k": False, + "selection_metrics_valid": selection_is_k_specific, + "interpretation": ( + "Selection is evaluated only at the declared single prediction k. " + "When multiple k values are requested without a declared k, metrics are recall-only; " + "separate k-specific prediction files are required for selection." + ), + }, + "scope": "candidate recall and candidate-in-set selection only; no browser task success", + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=Path, required=True, help="normalized Mind2Web rows JSONL") + parser.add_argument("--predictions", type=Path, required=True, help="retriever/selector predictions JSONL") + parser.add_argument("--output", type=Path, required=True, help="evaluation report JSON") + parser.add_argument("--split", help="evaluate only rows whose normalized split matches this value") + parser.add_argument("--k", type=int, nargs="+", default=[5, 10, 20, 32]) + args = parser.parse_args() + + report = evaluate_files(args.rows, args.predictions, args.k, split=args.split) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/tools/export_shared_bundle.py b/tools/export_shared_bundle.py new file mode 100644 index 0000000000000000000000000000000000000000..133fc75c0658dd6877308b6140d86dedb103e8ba --- /dev/null +++ b/tools/export_shared_bundle.py @@ -0,0 +1,348 @@ +"""Split the existing pilot graphs into one encoder and two unchanged heads. + +This is a graph partition, not retraining. Historical bundles are read-only. +The selected graph cuts are specific to the checked Vons pilot exports. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import math +import shutil +from collections import Counter +from pathlib import Path +from typing import Any + +MODEL_LIMIT_BYTES = 64 * 1024 * 1024 +ENCODER_INPUTS = ["input_ids", "attention_mask", "token_type_ids", "option_mask"] +EMBEDDINGS = "view_19" +POOLED = "div" + + +def digest(path: Path) -> str: + value = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + +def _onnx() -> Any: + import onnx + + return onnx + + +def _rename(model: Any, names: dict[str, str]) -> None: + for node in model.graph.node: + for fields in (node.input, node.output): + for index, name in enumerate(fields): + fields[index] = names.get(name, name) + for values in (model.graph.input, model.graph.output, model.graph.value_info, + model.graph.initializer): + for value in values: + value.name = names.get(value.name, value.name) + + +def _validate_external_paths(path: Path) -> None: + """Check external tensor locations before loading any external bytes.""" + onnx = _onnx() + model = onnx.load(path, load_external_data=False) + for tensor in model.graph.initializer: + if tensor.data_location != onnx.TensorProto.EXTERNAL: + continue + metadata = {item.key: item.value for item in tensor.external_data} + location = metadata.get("location", "") + relative = Path(location) + if not location or relative.is_absolute() or ".." in relative.parts: + raise ValueError(f"unsafe external tensor location: {location}") + target = path.parent / relative + if target.is_symlink() or not target.resolve(strict=True).is_relative_to(path.parent.resolve()): + raise ValueError(f"external tensor escapes graph directory: {location}") + + +def _tensor_hash(tensor: Any) -> str: + onnx = _onnx() + array = onnx.numpy_helper.to_array(tensor) + header = json.dumps([int(tensor.data_type), list(tensor.dims)]).encode() + return hashlib.sha256(header + array.tobytes()).hexdigest() + + +def encoder_parameters(model: Any) -> Counter[str]: + """Include named encoder tensors and exporter-folded parameter matrices. + + Exporter-folded matmul weights lose their original names. The pilot's + large floating point initializers are parameter matrices; positional/index + constants use integral dtypes. Small arithmetic constants are not weights. + """ + onnx = _onnx() + return Counter( + _tensor_hash(tensor) + for tensor in model.graph.initializer + if tensor.data_type == onnx.TensorProto.FLOAT + and (tensor.name.startswith("encoder.") or math.prod(tensor.dims) >= 4096) + ) + + +def partition_graphs(direct: Any, diffusion: Any) -> dict[str, Any]: + onnx = _onnx() + from onnx.utils import Extractor + + encoder = Extractor(direct).extract_model(ENCODER_INPUTS, [EMBEDDINGS, POOLED]) + other_encoder = Extractor(diffusion).extract_model(ENCODER_INPUTS, [EMBEDDINGS, POOLED]) + if not encoder_parameters(encoder) or encoder_parameters(encoder) != encoder_parameters(other_encoder): + raise ValueError("encoder parameter tensors differ; sharing would change the model") + direct_head = Extractor(direct).extract_model( + [EMBEDDINGS, POOLED, "option_mask"], ["logits", "answerability"], + ) + diffusion_head = Extractor(diffusion).extract_model( + [POOLED, "option_mask", "initial_noise"], ["scores", "answerability"], + ) + # The original fixed graph obtains its batch size from token_type_ids. + # The pooled representation has precisely the same leading batch axis. + for node in diffusion_head.graph.node: + if "token_type_ids" in node.input: + attributes = {item.name: onnx.helper.get_attribute_value(item) for item in node.attribute} + if node.op_type != "Shape" or attributes.get("start", 0) != 0 or attributes.get("end") != 1: + raise ValueError("unsupported token dependency in diffusion head") + for index, name in enumerate(node.input): + if name == "token_type_ids": + node.input[index] = POOLED + result = {"encoder": encoder, "direct": direct_head, "diffusion": diffusion_head} + for graph in result.values(): + _rename(graph, {EMBEDDINGS: "candidate_embeddings", POOLED: "pooled"}) + onnx.checker.check_model(graph) + return result + + +def export_bundle(direct_path: Path, diffusion_path: Path, tokenizer: Path, output: Path, + calibration: Path | None = None) -> dict[str, Any]: + onnx = _onnx() + if output.exists() and any(output.iterdir()): + raise ValueError("output directory must be new or empty; historical bundles are never overwritten") + for path in (direct_path, diffusion_path): + _validate_external_paths(path) + direct = onnx.load(direct_path) + diffusion = onnx.load(diffusion_path) + parts = partition_graphs(direct, diffusion) + output.mkdir(parents=True, exist_ok=True) + records = [] + for name, graph in parts.items(): + path = output / f"{name}.onnx" + onnx.save_model(copy.deepcopy(graph), path, save_as_external_data=True, + all_tensors_to_one_file=True, location=f"{name}.onnx.data", + size_threshold=1024, convert_attribute=False) + onnx.checker.check_model(str(path)) + records.append({"path": path.name, "role": f"{name}_graph"}) + external = path.with_suffix(".onnx.data") + if external.exists(): + records.append({"path": external.name, "role": f"{name}_weights"}) + token_dir = output / "tokenizer" + token_dir.mkdir() + for filename in ("tokenizer.json", "tokenizer_config.json"): + source = tokenizer / filename + if source.is_symlink() or not source.is_file(): + raise ValueError(f"required tokenizer asset missing or symlinked: {filename}") + shutil.copyfile(source, token_dir / filename) + records.append({"path": f"tokenizer/{filename}", "role": "tokenizer"}) + calibration_data: dict[str, Any] = {"direct": None, "diffusion": None} + calibration_source = None + if calibration is not None: + raw = json.loads(calibration.read_text()) + fit = raw.get("calibration", raw.get("config", {}).get("calibration")) + if not isinstance(fit, dict): + raise ValueError("direct calibration input has no fitted calibration parameters") + calibration_data["direct"] = fit + calibration_source = {"sha256": digest(calibration), "scope": "direct only"} + (output / "calibration.json").write_text(json.dumps(calibration_data, indent=2) + "\n") + records.append({"path": "calibration.json", "role": "calibration"}) + settings = { + "max_tokens": 512, "max_options": 32, "max_questions": 8, + "diffusion_inference_steps": 4, "diffusion_training_steps": 32, + "diffusion_candidate_slots": 32, "initial_noise": "host-supplied standard normal float32", + "default_abstain_threshold": 0.55, "default_answerability_threshold": 0.5, + "calibration": "calibration.json", "calibration_source": calibration_source, + } + (output / "config.json").write_text(json.dumps(settings, indent=2) + "\n") + records.append({"path": "config.json", "role": "config"}) + for record in records: + path = output / record["path"] + record.update(bytes=path.stat().st_size, sha256=digest(path)) + total = sum(record["bytes"] for record in records) + source_assets = [] + for label, source in (("direct", direct_path), ("diffusion", diffusion_path)): + source_names = {source.name} + source_model = onnx.load(source, load_external_data=False) + for tensor in source_model.graph.initializer: + source_names.update(item.value for item in tensor.external_data if item.key == "location") + source_assets.extend({"backend": label, "path": filename, + "bytes": (source.parent / filename).stat().st_size, + "sha256": digest(source.parent / filename)} + for filename in sorted(source_names)) + manifest = { + "schema": "vons.shared-bundle/v1", "model_id": "vons-pilot-shared-v1", + "transformation": "partition existing graphs; no training or parameter changes", + "encoder": {"graph": "encoder.onnx", "outputs": ["candidate_embeddings", "pooled"]}, + "heads": { + "direct": {"graph": "direct.onnx", "inputs": ["candidate_embeddings", "pooled", "option_mask"]}, + "diffusion": {"graph": "diffusion.onnx", "inputs": ["pooled", "option_mask", "initial_noise"]}, + }, + "files": records, "model_asset_bytes": total, "limit_bytes": MODEL_LIMIT_BYTES, + "size_pass": total <= MODEL_LIMIT_BYTES, + "source_graphs": {"direct_sha256": digest(direct_path), "diffusion_sha256": digest(diffusion_path)}, + "source_assets": source_assets, + "source_tool_sha256": digest(Path(__file__)), + "release_status": "research_candidate; parity and notices required before release", + "runtime_bytes": None, "runtime_bytes_reason": "runtime is not included in this model-only bundle", + } + manifest_path = output / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + (output / "manifest.sha256").write_text(f"{digest(manifest_path)} manifest.json\n") + verify_manifest(manifest_path) + return manifest + + +def verify_manifest(path: Path, *, expected_digest: str | None = None) -> dict[str, Any]: + manifest = json.loads(path.read_text()) + if manifest.get("schema") != "vons.shared-bundle/v1": + raise ValueError("unsupported shared bundle manifest") + expected = expected_digest or path.with_suffix(".sha256").read_text().split()[0] + if digest(path) != expected: + raise ValueError("release manifest digest mismatch") + root = path.parent.resolve() + names = set() + total = 0 + for item in manifest["files"]: + relative = Path(item["path"]) + target = root / relative + if relative.is_absolute() or ".." in relative.parts or target.is_symlink(): + raise ValueError("unsafe manifest asset path") + if not target.resolve(strict=True).is_relative_to(root): + raise ValueError("manifest asset escapes bundle") + if relative.as_posix() in names: + raise ValueError("duplicate manifest asset") + names.add(relative.as_posix()) + if target.stat().st_size != item["bytes"] or digest(target) != item["sha256"]: + raise ValueError(f"asset digest or byte count mismatch: {relative}") + total += target.stat().st_size + if total != manifest["model_asset_bytes"] or manifest["size_pass"] != (total <= MODEL_LIMIT_BYTES): + raise ValueError("bundle byte total mismatch") + required = {"encoder.onnx", "direct.onnx", "diffusion.onnx", "config.json", + "calibration.json", "tokenizer/tokenizer.json", "tokenizer/tokenizer_config.json"} + if not required.issubset(names): + raise ValueError("manifest omits required model assets") + onnx = _onnx() + for filename in ("encoder.onnx", "direct.onnx", "diffusion.onnx"): + _validate_external_paths(root / filename) + graph = onnx.load(root / filename, load_external_data=False) + for tensor in graph.graph.initializer: + for item in tensor.external_data: + if item.key == "location" and item.value not in names: + raise ValueError("manifest omits referenced external tensor data") + return {"pass": True, "model_asset_bytes": total, "size_pass": total <= MODEL_LIMIT_BYTES, + "manifest_sha256": digest(path)} + + +def verify_parity(bundle: Path, direct_path: Path, diffusion_path: Path, + *, cases: int = 20) -> dict[str, Any]: + """Compare graph partitions with historical graphs using identical tensors. + + This verifies graph transformation only. It is not task-quality evaluation + or a latency benchmark, and does not validate browser tokenization. + """ + import numpy as np + import onnxruntime as ort + + verified = verify_manifest(bundle / "manifest.json") + options = ort.SessionOptions() + options.intra_op_num_threads = 2 + options.inter_op_num_threads = 1 + options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + + def session(path: Path) -> Any: + return ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"]) + + encoder = session(bundle / "encoder.onnx") + heads = {name: session(bundle / f"{name}.onnx") for name in ("direct", "diffusion")} + full = {"direct": session(direct_path), "diffusion": session(diffusion_path)} + samples = [] + for case in range(cases): + count = (2, 4, 8, 16, 32)[case % 5] + batch = 2 if case % 4 == 3 else 1 + length = (8, 31, 64, 127)[case % 4] + # Token IDs are valid BERT vocabulary entries. Fixed local arrays avoid + # downloads and isolate graph partitioning from tokenizer differences. + random = np.random.default_rng(700 + case) + tokens = random.integers(100, 2000, (batch, count, length), dtype=np.int64) + mask = np.zeros((batch, 32), dtype=np.bool_) + mask[:, :count] = True + ids = np.zeros((batch, 32, 512), np.int64) + attention = np.zeros_like(ids) + ids[:, :count, :length] = tokens + attention[:, :count, :length] = 1 + feed = {"input_ids": ids, "attention_mask": attention, + "token_type_ids": np.zeros_like(ids), "option_mask": mask} + representation = encoder.run(None, feed) + for backend in ("direct", "diffusion"): + for seed in ((7, 17, 27) if backend == "diffusion" else (7,)): + noise = np.random.default_rng(seed + case).standard_normal((batch, 32)).astype(np.float32) + shared_feed = {"pooled": representation[1], "option_mask": mask} + original_feed = dict(feed) + if backend == "direct": + shared_feed["candidate_embeddings"] = representation[0] + else: + shared_feed["initial_noise"] = noise + original_feed["initial_noise"] = noise + expected = full[backend].run(None, original_feed) + actual = heads[backend].run(None, shared_feed) + finite = np.broadcast_to(mask, expected[0].shape) + scores_finite = bool(np.isfinite(expected[0][finite]).all() + and np.isfinite(actual[0][finite]).all()) + masked = bool(np.isneginf(expected[0][~finite]).all() + and np.isneginf(actual[0][~finite]).all()) + answers_finite = bool(np.isfinite(expected[1]).all() and np.isfinite(actual[1]).all()) + passed = bool(scores_finite and masked and answers_finite + and np.allclose(expected[0][finite], actual[0][finite], atol=1e-5, rtol=1e-5) + and np.allclose(expected[1], actual[1], atol=1e-5, rtol=1e-5)) + samples.append({ + "case": case, "backend": backend, "seed": seed, "candidates": count, + "batch": batch, "shape": list(ids.shape), "active_tokens": length, + "input_sha256": hashlib.sha256(ids.tobytes() + attention.tobytes() + mask.tobytes()).hexdigest(), + "noise_sha256": hashlib.sha256(noise.tobytes()).hexdigest() if backend == "diffusion" else None, + "max_abs_scores": float(np.max(np.abs(expected[0][finite] - actual[0][finite]))), + "max_abs_answerability": float(np.max(np.abs(expected[1] - actual[1]))), + "pass": passed, + }) + return {"schema": "vons.shared-bundle-parity/v1", "manifest_sha256": verified["manifest_sha256"], + "provider": "CPUExecutionProvider", "ort_version": ort.__version__, + "intra_op_threads": 2, "atol": 1e-5, "rtol": 1e-5, "cases": cases, + "scope": "graph partition only; no quality, browser, calibration or latency claim", + "samples": samples, "pass": all(sample["pass"] for sample in samples)} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--direct", type=Path, required=True) + parser.add_argument("--diffusion", type=Path, required=True) + parser.add_argument("--tokenizer", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--direct-calibration", type=Path) + parser.add_argument("--parity-report", type=Path) + args = parser.parse_args() + report = export_bundle(args.direct, args.diffusion, args.tokenizer, args.output, + args.direct_calibration) + print(json.dumps({key: report[key] for key in ("model_asset_bytes", "size_pass", "release_status")})) + if args.parity_report is not None: + parity = verify_parity(args.output, args.direct, args.diffusion) + args.parity_report.parent.mkdir(parents=True, exist_ok=True) + args.parity_report.write_text(json.dumps(parity, indent=2, allow_nan=False) + "\n") + print(json.dumps({"parity_pass": parity["pass"], "samples": len(parity["samples"])})) + if not parity["pass"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/freeze_publication_packet.py b/tools/freeze_publication_packet.py new file mode 100644 index 0000000000000000000000000000000000000000..439af1f9e1a201d3b1321d3efb7395afdbd6d394 --- /dev/null +++ b/tools/freeze_publication_packet.py @@ -0,0 +1,79 @@ +"""Freeze an explicit evidence/source allowlist for both manuscript authors. + +The packet contains copies of reports and source, never model weights or raw +restricted test data. Its manifest authenticates copies relative to the digest +delivered separately to the authors; it does not certify experimental quality. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import tempfile +import zipfile +from pathlib import Path + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def freeze(spec_path: Path, output: Path) -> dict: + root = Path(__file__).resolve().parents[1] + spec = json.loads(spec_path.read_text()) + if output.exists() or output.with_suffix(".zip").exists(): + raise ValueError("refusing to overwrite a frozen packet") + inputs = spec["inputs"] + if not inputs or len({item["path"] for item in inputs}) != len(inputs): + raise ValueError("packet requires a unique explicit input list") + output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=output.parent) as temporary: + stage = Path(temporary) / "packet" + stage.mkdir() + entries = [] + for item in inputs: + relative = Path(item["path"]) + source = root / relative + if (relative.is_absolute() or ".." in relative.parts + or not source.resolve().is_relative_to(root) + or relative.parts[:2] == ("data", "restricted") + or relative.suffix in {".pt", ".onnx", ".data", ".safetensors", ".pyc"}): + raise ValueError(f"not an allowed evidence path: {relative}") + digest = sha256(source) + if item.get("sha256") != digest: + raise ValueError(f"input changed since the specification: {relative}") + target = stage / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + if sha256(target) != digest: + raise ValueError(f"copy mismatch: {relative}") + entries.append({"path": relative.as_posix(), "role": item["role"], + "bytes": target.stat().st_size, "sha256": digest}) + packet = {"schema": "vons.publication-evidence/v2", "project": "Vons", + "git_revision": None, "git_revision_reason": "workspace has no Git repository", + "spec_sha256": sha256(spec_path), "context": spec["context"], "files": entries} + (stage / "packet.json").write_text(json.dumps(packet, indent=2, allow_nan=False) + "\n") + shutil.copyfile(spec_path, stage / "input-spec.json") + for entry in entries: + if sha256(root / entry["path"]) != entry["sha256"]: + raise ValueError("source changed during freeze") + stage.rename(output) + archive = output.with_suffix(".zip") + with zipfile.ZipFile(archive, "x", compression=zipfile.ZIP_DEFLATED) as bundle: + for path in sorted(output.rglob("*")): + if path.is_file(): + entry = zipfile.ZipInfo(path.relative_to(output).as_posix(), (2026, 1, 1, 0, 0, 0)) + entry.compress_type = zipfile.ZIP_DEFLATED + bundle.writestr(entry, path.read_bytes()) + return {"directory": str(output), "archive": str(archive), "archive_sha256": sha256(archive), + "packet_sha256": sha256(output / "packet.json"), "files": len(entries)} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spec", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(freeze(args.spec, args.output), indent=2)) diff --git a/tools/import_mind2web.py b/tools/import_mind2web.py new file mode 100644 index 0000000000000000000000000000000000000000..0921b7661322164e27e71f1363c8826e54949fff --- /dev/null +++ b/tools/import_mind2web.py @@ -0,0 +1,635 @@ +"""Import the pinned official Mind2Web held-out tasks for local evaluation only. + +Outputs are restricted: rows.jsonl contains labels; retriever.jsonl contains +only pre-action inputs and deterministic lexical ranks. Nothing executes a +browser action. Current operation/value and current/future action_reprs are +never model inputs. The history is the official, teacher-forced past history. + +Example (large extraction/normalization requires an available compute window):: + + python tools/import_mind2web.py --input-dir data/restricted/mind2web/extracted \ + --split all --output-dir data/restricted/mind2web/normalized \ + --source-manifest data/restricted/mind2web/acquisition-manifest.json \ + --extract-archive data/restricted/mind2web/acquisition/test.zip + +Reusing an extracted corpus omits --extract-archive. The source manifest must +be the acquisition record for the pinned ZIP; file hashes are filled during +bounded extraction. Existing outputs are never overwritten. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +import shutil +import stat +import tempfile +import zipfile +from collections import Counter +from collections.abc import Iterator, Mapping, Sequence +from pathlib import Path, PurePosixPath +from typing import TextIO +from xml.etree.ElementTree import ParseError, XMLParser + +DATASET_REVISION = "17ece8eb89862368edc0cc806acee6fca5163474" +ARCHIVE_SHA256 = "8f5fbe72afab942fe97cdf7fb397e179885d89b5c16862288e9a14bc6d41ca89" +ARCHIVE_BYTES = 567745122 +OFFICIAL_TASK_COUNTS = {"test_task": 252, "test_website": 177, "test_domain": 912} +OFFICIAL_FILE_COUNTS = {"test_task": 3, "test_website": 2, "test_domain": 10} +SPLITS = tuple(OFFICIAL_TASK_COUNTS) +MAX_ARCHIVE_BYTES = 1024**3 +MAX_MEMBER_BYTES = 1024**3 +MAX_EXTRACTED_BYTES = 8 * 1024**3 +# Some official tasks contain over 128 Mi characters, mostly unused raw_html. +# A task cannot exceed its verified member's byte bound; do not cap it below that. +MAX_TASK_CHARS = MAX_MEMBER_BYTES +CHUNK_SIZE = 1024**2 +# Allowlisted observable DOM attributes exclude annotation flags by construction. +TEXT_ATTRIBUTES = frozenset( + { + "alt", + "aria_description", + "aria_label", + "aria_role", + "class", + "href", + "id", + "input_checked", + "input_value", + "label", + "name", + "option_selected", + "placeholder", + "role", + "text_value", + "title", + "type", + "value", + } +) + + +def _object(value: object, label: str) -> Mapping[str, object]: + if not isinstance(value, dict): + raise TypeError(f"{label} must be an object") + return value + + +def _array(value: object, label: str) -> list[object]: + if not isinstance(value, list): + raise TypeError(f"{label} must be an array") + return value + + +def _text(value: object, label: str, *, allow_empty: bool = False) -> str: + if not isinstance(value, str) or (not value.strip() and not allow_empty): + raise ValueError(f"{label} must be a string") + return value + + +def _identifier(value: object, label: str) -> str: + value = _text(value, label) + if not re.fullmatch(r"[A-Za-z0-9_.-]+", value): + raise ValueError(f"{label} contains unsupported identifier characters") + return value + + +def _canonical(value: object) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(CHUNK_SIZE): + digest.update(chunk) + return digest.hexdigest() + + +def _json_hash(value: object) -> str: + return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest() + + +def _clean(value: str) -> str: + return " ".join(value.split()) + + +def _attributes(value: Mapping[str, object]) -> dict[str, str]: + result: dict[str, str] = {} + for name, raw in value.items(): + name = name.lower().replace("-", "_") + if name in TEXT_ATTRIBUTES and isinstance(raw, (str, bool, int, float)): + cleaned = _clean(str(raw)) + if cleaned: + result[name] = cleaned + return result + + +class _CandidateDOM: + """XML parser target for pre-action DOM nodes, including iframe descendants. + + The official cleaned DOM is XML serialization. Browser HTML parsing treats + iframe contents as raw text and silently loses released candidate nodes. + """ + + def __init__(self, candidate_ids: set[str]) -> None: + self.candidate_ids = candidate_ids + self.stack: list[tuple[str, str | None]] = [] + self.nodes: dict[str, tuple[str, dict[str, str], list[str]]] = {} + + def start(self, tag: str, attrs: dict[str, str]) -> None: + self._boundary() + node_id = attrs.get("backend_node_id") + candidate_id = node_id if node_id in self.candidate_ids else None + if candidate_id is not None: + if candidate_id in self.nodes: + raise ValueError("duplicate candidate node id in cleaned DOM") + self.nodes[candidate_id] = (tag, _attributes(attrs), []) + self.stack.append((tag, candidate_id)) + + def end(self, tag: str) -> None: + if self.stack[-1][0] != tag: + raise ValueError("unbalanced cleaned DOM") + self._boundary() + self.stack.pop() + + def _boundary(self) -> None: + for _, candidate_id in self.stack: + if candidate_id is not None: + self.nodes[candidate_id][2].append(" ") + + def data(self, data: str) -> None: + if any(tag in {"script", "style"} for tag, _ in self.stack): + return + if data: + for _, candidate_id in self.stack: + if candidate_id is not None: + self.nodes[candidate_id][2].append(data) + + def close(self) -> None: + return None + + +def candidate_universe(action: Mapping[str, object]) -> list[dict[str, str]]: + """Merge the released pool without exposing annotation membership or order.""" + candidates: dict[str, Mapping[str, object]] = {} + for field in ("pos_candidates", "neg_candidates"): + for raw in _array(action.get(field), field): + candidate = _object(raw, "candidate") + node_id = _identifier(candidate.get("backend_node_id"), "backend_node_id") + if node_id in candidates: + raise ValueError("duplicate candidate id in released pool") + candidates[node_id] = candidate + cleaned_html = _text(action.get("cleaned_html"), "cleaned_html", allow_empty=True) + dom = _CandidateDOM(set(candidates)) + if cleaned_html: + parser = XMLParser(target=dom) + try: + parser.feed(cleaned_html) + parser.close() + except ParseError: + raise ValueError("cleaned DOM is not valid XML serialization") from None + if set(dom.nodes) != set(candidates): + raise ValueError("released candidate missing from cleaned DOM") + result: list[dict[str, str]] = [] + for node_id in sorted(candidates): + tag, dom_attributes, content = dom.nodes[node_id] + raw_attributes = candidates[node_id].get("attributes", "{}") + if isinstance(raw_attributes, str): + try: + raw_attributes = json.loads(raw_attributes) + except json.JSONDecodeError: + raise ValueError("invalid candidate attributes JSON") from None + attributes = _attributes(_object(raw_attributes, "candidate attributes")) + attributes.update(dom_attributes) + parts = [tag, *(f"{key}={attributes[key]}" for key in sorted(attributes)), "".join(content)] + result.append({"id": node_id, "text": _clean(" ".join(parts))}) + return result + + +def lexical_rank(query: str, candidates: Sequence[Mapping[str, str]]) -> list[dict[str, object]]: + """Deterministic BM25 (k1=1.2, b=0.75); the interface has no labels.""" + ids = [candidate["id"] for candidate in candidates] + if len(ids) != len(set(ids)): + raise ValueError("lexical candidates must have unique ids") + query_terms = set(re.findall(r"\w+", query.casefold())) + terms = [Counter(re.findall(r"\w+", candidate["text"].casefold())) for candidate in candidates] + lengths = [sum(row.values()) for row in terms] + average_length = sum(lengths) / len(lengths) if lengths else 1.0 + document_frequency: Counter[str] = Counter() + for row in terms: + document_frequency.update(set(row)) + ranked: list[dict[str, object]] = [] + for candidate, counts, length in zip(candidates, terms, lengths): + score = 0.0 + for term in sorted(query_terms): + frequency = counts[term] + if frequency: + inverse_frequency = math.log1p( + (len(candidates) - document_frequency[term] + 0.5) + / (document_frequency[term] + 0.5) + ) + denominator = frequency + 1.2 * (0.25 + 0.75 * length / average_length) + score += inverse_frequency * frequency * 2.2 / denominator + ranked.append({"id": candidate["id"], "text": candidate["text"], "score": score}) + ranked.sort(key=lambda row: (-float(row["score"]), str(row["id"]))) + return [{**row, "rank": rank} for rank, row in enumerate(ranked, start=1)] + + +def normalize_task( + task: Mapping[str, object], + split: str, +) -> Iterator[tuple[dict[str, object], dict[str, object]]]: + if split not in SPLITS: + raise ValueError("only official held-out splits are supported") + if "split" in task and task["split"] != split: + raise ValueError("task split disagrees with official source file") + task_id = _identifier(task.get("annotation_id"), "annotation_id") + website = _text(task.get("website"), "website") + domain = _text(task.get("domain"), "domain") + goal = _text(task.get("confirmed_task"), "confirmed_task") + actions = _array(task.get("actions"), "actions") + history = _array(task.get("action_reprs"), "action_reprs") + if not actions or len(history) != len(actions): + raise ValueError("actions and action_reprs must be nonempty and aligned") + seen_actions: set[str] = set() + for index, raw_action in enumerate(actions): + action = _object(raw_action, "action") + action_id = _identifier(action.get("action_uid"), "action_uid") + if action_id in seen_actions: + raise ValueError("duplicate action id within task") + seen_actions.add(action_id) + candidates = candidate_universe(action) + previous_actions = [_text(item, "past action_repr") for item in history[:index]] + state = {"goal": goal, "previous_actions": previous_actions} + query = goal + ( + "\nPrevious actions:\n" + "\n".join(previous_actions) if previous_actions else "" + ) + ranked = lexical_rank(query, candidates) + identity = { + "id": f"{task_id}:{action_id}", + "task_id": task_id, + "action_id": action_id, + "split": split, + "website": website, + "domain": domain, + } + # Membership is consulted only after label-blind inputs/ranks are complete. + positive_ids = sorted( + _identifier( + _object(item, "positive candidate").get("backend_node_id"), "backend_node_id" + ) + for item in _array(action.get("pos_candidates"), "pos_candidates") + ) + labels = { + **identity, + "candidate_ids": [item["id"] for item in candidates], + "positive_ids": positive_ids, + "no_positive": not positive_ids, + } + retrieval = { + **identity, + "state": state, + "query": query, + "candidates": ranked, + "generated_candidates": [item["id"] for item in ranked], + "input_sha256": _json_hash({"state": state, "candidates": candidates}), + } + yield labels, retrieval + + +def iter_tasks(handle: TextIO) -> Iterator[Mapping[str, object]]: + """Decode one task at a time from an official JSON array with a size bound.""" + decoder = json.JSONDecoder() + buffer = "" + eof = False + + def more(read_size: int = CHUNK_SIZE) -> bool: + nonlocal buffer, eof + chunk = handle.read(read_size) + eof = not chunk + buffer += chunk + if len(buffer) > MAX_TASK_CHARS: + raise ValueError("task JSON exceeds configured buffer bound") + return bool(chunk) + + def strip_space() -> None: + nonlocal buffer + buffer = buffer.lstrip() + while not buffer and not eof: + more() + buffer = buffer.lstrip() + + strip_space() + if not buffer.startswith("["): + raise ValueError("official task file must be a JSON array") + buffer = buffer[1:] + strip_space() + if buffer.startswith("]"): + buffer = buffer[1:] + else: + while True: + read_size = CHUNK_SIZE + while True: + try: + task, end = decoder.raw_decode(buffer) + break + except json.JSONDecodeError: + if eof or not more(read_size): + raise ValueError("invalid or truncated task JSON") from None + # Re-decoding an incomplete large task every MiB is quadratic. + read_size = min(read_size * 2, MAX_TASK_CHARS) + yield _object(task, "task") + buffer = buffer[end:] + strip_space() + if buffer.startswith("]"): + buffer = buffer[1:] + break + if not buffer.startswith(","): + raise ValueError("invalid task array separator") + buffer = buffer[1:] + strip_space() + strip_space() + if buffer: + raise ValueError("trailing content after task array") + + +def _member_split(name: str) -> str: + path = PurePosixPath(name) + if ( + path.is_absolute() + or len(path.parts) != 2 + or "\\" in name + or ".." in path.parts + or path.as_posix() != name + ): + raise ValueError("unsafe or unexpected archive member path") + split = path.parts[0] + if split not in SPLITS or not re.fullmatch(rf"{split}_\d+\.json", path.name): + raise ValueError("archive member has unknown split or filename") + return split + + +def _validate_manifest(manifest: Mapping[str, object]) -> Mapping[str, object]: + archive = _object(manifest.get("archive"), "archive") + if ( + manifest.get("schema") != "vons.mind2web-acquisition/v1" + or manifest.get("dataset") != "osunlp/Mind2Web" + or manifest.get("dataset_revision") != DATASET_REVISION + or archive.get("sha256") != ARCHIVE_SHA256 + or archive.get("bytes") != ARCHIVE_BYTES + or archive.get("status") != "downloaded-and-hash-verified" + ): + raise ValueError("source manifest does not match the pinned official test archive") + restrictions = _object(manifest.get("restrictions"), "restrictions") + if ( + manifest.get("usage") != "evaluation-only" + or restrictions.get("no_training") is not True + or restrictions.get("no_unzipped_redistribution") is not True + ): + raise ValueError("source manifest must preserve evaluation-only usage restrictions") + return archive + + +def validate_archive_members(infos: Sequence[zipfile.ZipInfo]) -> None: + expected_names = { + f"{split}/{split}_{index}.json" + for split, count in OFFICIAL_FILE_COUNTS.items() + for index in range(count) + } + names: set[str] = set() + total = 0 + for info in infos: + _member_split(info.filename) + mode = info.external_attr >> 16 + if info.is_dir() or stat.S_ISLNK(mode): + raise ValueError("archive contains a directory or symlink") + if info.filename in names: + raise ValueError("archive contains a duplicate member") + names.add(info.filename) + if not 0 < info.file_size <= MAX_MEMBER_BYTES: + raise ValueError("archive member exceeds extraction size bound") + total += info.file_size + if names != expected_names or total > MAX_EXTRACTED_BYTES: + raise ValueError("archive member set or total size differs from official bounds") + + +def extract_official_archive(archive_path: Path, input_dir: Path, manifest_path: Path) -> None: + manifest = dict(_object(json.loads(manifest_path.read_text()), "source manifest")) + archive = _validate_manifest(manifest) + if archive_path.stat().st_size > MAX_ARCHIVE_BYTES: + raise ValueError("archive exceeds download bound") + if ( + archive_path.stat().st_size != archive["bytes"] + or _sha256(archive_path) != archive["sha256"] + ): + raise ValueError("archive failed pinned size/hash verification") + if input_dir.exists(): + raise ValueError("refusing to overwrite existing extraction directory") + input_dir.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(archive_path) as source: + infos = sorted(source.infolist(), key=lambda info: info.filename) + validate_archive_members(infos) + if shutil.disk_usage(input_dir.parent).free < sum(info.file_size for info in infos): + raise ValueError("insufficient free disk for bounded extraction") + with tempfile.TemporaryDirectory(prefix=".mind2web-extract-", dir=input_dir.parent) as temp: + staging = Path(temp) / "data" + staging.mkdir() + records: list[dict[str, object]] = [] + for info in infos: + path = staging / info.filename + path.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + size = 0 + with source.open(info, pwd=b"mind2web") as reader, path.open("xb") as writer: + while chunk := reader.read(CHUNK_SIZE): + size += len(chunk) + if size > info.file_size: + raise ValueError("decompressed member exceeded declared size") + writer.write(chunk) + digest.update(chunk) + if size != info.file_size: + raise ValueError("decompressed member size mismatch") + records.append( + { + "path": info.filename, + "split": _member_split(info.filename), + "bytes": size, + "sha256": digest.hexdigest(), + } + ) + staging.rename(input_dir) + manifest["files"] = records + manifest["extraction"] = { + "status": "complete", + "file_count": len(records), + "uncompressed_bytes": sum(int(row["bytes"]) for row in records), + } + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + +def import_split( + input_dir: Path, + splits: Sequence[str], + output_dir: Path, + manifest_path: Path, +) -> dict[str, object]: + if ( + not splits + or len(splits) != len(set(splits)) + or any(split not in SPLITS for split in splits) + ): + raise ValueError("split selection must contain distinct official held-out split names") + manifest = _object(json.loads(manifest_path.read_text(encoding="utf-8")), "source manifest") + _validate_manifest(manifest) + if _object(manifest.get("extraction"), "extraction").get("status") != "complete": + raise ValueError("official archive extraction is incomplete") + records = [_object(row, "file record") for row in _array(manifest.get("files"), "files")] + paths: set[str] = set() + for record in records: + name = _text(record.get("path"), "source path") + split = _member_split(name) + if name in paths or record.get("split") != split: + raise ValueError("duplicate source path or incorrect official split identity") + paths.add(name) + expected = { + f"{split}/{split}_{index}.json" + for split, count in OFFICIAL_FILE_COUNTS.items() + for index in range(count) + } + if paths != expected: + raise ValueError("source file manifest does not cover the official held-out corpus") + selected = sorted( + (row for row in records if row["split"] in splits), key=lambda row: str(row["path"]) + ) + if output_dir.exists(): + raise ValueError("refusing to overwrite existing output directory") + input_root = input_dir.resolve() + for record in selected: + path = input_dir / str(record["path"]) + if path.is_symlink() or not path.resolve().is_relative_to(input_root): + raise ValueError("source file resolves outside the input directory") + if path.stat().st_size != record.get("bytes") or _sha256(path) != record.get("sha256"): + raise ValueError("source file failed size/hash verification") + output_dir.parent.mkdir(parents=True, exist_ok=True) + task_counts: Counter[str] = Counter() + row_counts: Counter[str] = Counter() + no_positive_counts: Counter[str] = Counter() + seen_tasks: set[str] = set() + seen_actions: set[str] = set() + with tempfile.TemporaryDirectory(prefix=".mind2web-import-", dir=output_dir.parent) as temp: + staging = Path(temp) / "data" + staging.mkdir() + rows_path = staging / "rows.jsonl" + retriever_path = staging / "retriever.jsonl" + with ( + rows_path.open("w", encoding="utf-8") as labels_out, + retriever_path.open("w", encoding="utf-8") as retrieval_out, + ): + for record in selected: + split = str(record["split"]) + with (input_dir / str(record["path"])).open(encoding="utf-8") as handle: + for task in iter_tasks(handle): + task_id = _identifier(task.get("annotation_id"), "annotation_id") + if task_id in seen_tasks: + raise ValueError("duplicate task id across input files or splits") + seen_tasks.add(task_id) + task_counts[split] += 1 + for labels, retrieval in normalize_task(task, split): + action_id = str(labels["action_id"]) + if action_id in seen_actions: + raise ValueError("duplicate action id across tasks or splits") + seen_actions.add(action_id) + provenance = { + "dataset_revision": DATASET_REVISION, + "source_file": record["path"], + "source_sha256": record["sha256"], + } + labels["provenance"] = provenance + retrieval["provenance"] = provenance + labels_out.write(_canonical(labels) + "\n") + retrieval_out.write(_canonical(retrieval) + "\n") + row_counts[split] += 1 + no_positive_counts[split] += int(bool(labels["no_positive"])) + if any(task_counts[split] != OFFICIAL_TASK_COUNTS[split] for split in splits): + raise ValueError("task counts do not match complete official held-out splits") + report: dict[str, object] = { + "schema": "vons.mind2web-import/v1", + "dataset_revision": DATASET_REVISION, + "archive_sha256": ARCHIVE_SHA256, + "source_manifest_sha256": _sha256(manifest_path), + "importer_sha256": _sha256(Path(__file__)), + "splits": list(splits), + "task_counts": dict(task_counts), + "row_counts": dict(row_counts), + "no_positive_counts": dict(no_positive_counts), + "source_files": selected, + "outputs": [ + {"path": path.name, "sha256": _sha256(path), "bytes": path.stat().st_size} + for path in (rows_path, retriever_path) + ], + "retriever": { + "name": "lexical-bm25", + "k1": 1.2, + "b": 0.75, + "tokenization": "Unicode word tokens, casefold, unique query terms", + "tie_break": "lexicographic candidate id", + "top_k": None, + }, + "candidate_order": "lexicographic candidate id after merging the released pool", + "history": "teacher-forced official action_reprs strictly before the current action", + "usage": "evaluation-only; no training or public unzipped redistribution", + "scope": "official held-out candidate retrieval; no model or browser task evaluation", + "limitations": [ + "Released candidate-pool recall bounds the task; this does not generate DOM candidates.", + "No training split was acquired; train/test identity overlap is not independently audited.", + "Duplicate identities are checked across the selected splits only.", + ], + } + (staging / "manifest.json").write_text( + json.dumps(report, indent=2) + "\n", encoding="utf-8" + ) + (staging / ".gitignore").write_text("*\n", encoding="utf-8") + staging.rename(output_dir) + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-dir", type=Path, required=True) + parser.add_argument("--split", nargs="+", choices=[*SPLITS, "all"], required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--source-manifest", type=Path, required=True) + parser.add_argument("--extract-archive", type=Path) + args = parser.parse_args() + if "all" in args.split and args.split != ["all"]: + parser.error("all cannot be combined with another split") + try: + if args.extract_archive: + extract_official_archive(args.extract_archive, args.input_dir, args.source_manifest) + report = import_split( + args.input_dir, + SPLITS if args.split == ["all"] else args.split, + args.output_dir, + args.source_manifest, + ) + except (ValueError, TypeError, OSError, zipfile.BadZipFile, RuntimeError) as exc: + # Deliberately omit source text and JSON decoder excerpts from public logs. + parser.exit(2, f"Mind2Web import failed ({type(exc).__name__}); no evaluation claimed.\n") + print( + json.dumps( + { + "status": "imported", + "task_counts": report["task_counts"], + "row_counts": report["row_counts"], + "scope": report["scope"], + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tools/merge_browser_benchmarks.py b/tools/merge_browser_benchmarks.py new file mode 100644 index 0000000000000000000000000000000000000000..48966f26fee2cef94622409520579aa6ced7805d --- /dev/null +++ b/tools/merge_browser_benchmarks.py @@ -0,0 +1,128 @@ +"""Merge independently saved one-session browser benchmark reports. + +Running one fresh ORT session per page/report avoids conflating session +teardown and reinitialization failures with the timed request population. The +merged report is still passed through the strict raw-sample verifier. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +from verify_browser_benchmark import file_sha256, percentile, verify_report + + +def _read_report(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError(f"{path} must contain a JSON object") + return value + + +def _summary(samples: list[dict[str, Any]]) -> dict[str, Any]: + successful = [sample for sample in samples if sample["status"] == "ok"] + failed = len(samples) - len(successful) + + def values(field: str) -> list[float]: + return [float(sample[field]) for sample in successful] + + def memory_values(field: str) -> list[float]: + return [float(sample[field]) for sample in successful if sample.get(field) is not None] + + before = memory_values("js_heap_used_bytes_before") + after = memory_values("js_heap_used_bytes_after") + return { + "successful_samples": len(successful), + "failed_samples": failed, + "total_request_ms_p50": percentile(values("totalRequestMs"), 0.5), + "total_request_ms_p95": percentile(values("totalRequestMs"), 0.95), + "inference_and_readback_ms_p50": percentile(values("inferenceAndReadbackMs"), 0.5), + "inference_and_readback_ms_p95": percentile(values("inferenceAndReadbackMs"), 0.95), + "js_heap_before_max": max(before) if before else None, + "js_heap_after_max": max(after) if after else None, + "memory_note": "performance.memory before/after observations only; not a peak or GPU-memory measurement", + } + + +def merge_reports(paths: list[Path]) -> dict[str, Any]: + if len(paths) < 2: + raise ValueError("at least two one-session reports are required") + raw_reports = [_read_report(path) for path in paths] + for path in paths: + verify_report(path) + + baseline = raw_reports[0] + comparable_keys = ( + "backend", + "requested_provider", + "warmup_excluded_per_session", + "repeats_per_session", + "cases", + "environment", + "runtime_info", + "execution_provider_evidence", + ) + for index, report in enumerate(raw_reports, start=1): + if report.get("sessions") != 1: + raise ValueError(f"source report {index} must have sessions=1") + for key in comparable_keys: + if report.get(key) != baseline.get(key): + raise ValueError(f"source report {index} does not match baseline field {key}") + + source_hashes = [file_sha256(path) for path in paths] + merged_run_id = "merged-" + hashlib.sha256("\n".join(source_hashes).encode()).hexdigest()[:16] + samples: list[dict[str, Any]] = [] + session_records: list[dict[str, Any]] = [] + for session_start, report in enumerate(raw_reports, start=1): + record = dict(report["session_records"][0]) + record["session_start"] = session_start + session_records.append(record) + for raw_sample in report["samples"]: + sample = dict(raw_sample) + sample["source_run_id"] = sample.get("run_id") + sample["run_id"] = merged_run_id + sample["session_start"] = session_start + samples.append(sample) + + return { + "schema": "vons.browser-benchmark/v1", + "run_id": merged_run_id, + "backend": baseline["backend"], + "requested_provider": baseline["requested_provider"], + "sessions": len(raw_reports), + "warmup_excluded_per_session": baseline["warmup_excluded_per_session"], + "repeats_per_session": baseline["repeats_per_session"], + "cases": baseline["cases"], + "session_start_semantics": "one fresh page/session per source report; browser caches may persist", + "execution_provider_evidence": baseline["execution_provider_evidence"], + "environment": baseline["environment"], + "runtime_info": baseline.get("runtime_info"), + "merge": { + "source_count": len(paths), + "source_reports": [ + {"path": str(path), "sha256": digest} for path, digest in zip(paths, source_hashes, strict=True) + ], + }, + "session_records": session_records, + "samples": samples, + "summary": _summary(samples), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report", type=Path, action="append", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + merged = merge_reports(args.report) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(merged, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tools/predict_mind2web.py b/tools/predict_mind2web.py new file mode 100644 index 0000000000000000000000000000000000000000..a4a586a27979f72de77c90576fe48f77ba7bc864 --- /dev/null +++ b/tools/predict_mind2web.py @@ -0,0 +1,241 @@ +"""Generate real, separate k-specific Mind2Web selections without reading labels. + +Strict 512-token question budget: overflow is an observed error, never silently +truncated. Outputs stay local because candidate text comes from restricted data. +The lexical baseline bypasses Vons' input contract and is labeled accordingly. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +import re +import time +from collections import Counter +from collections.abc import Callable +from pathlib import Path +from typing import Any + +QUESTION = "Which element should the agent interact with next to achieve the goal?" +KS = (5, 10, 20, 32) + + +class ModelOutputError(ValueError): + """Invalid numerical evidence stops the comparison, preserving its journal.""" + + +def file_hash(path: Path) -> str: + value = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + + +def canonical(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), + allow_nan=False) + + +def object_hash(value: Any) -> str: + return hashlib.sha256(canonical(value).encode()).hexdigest() + + +def make_request(row: dict[str, Any], k: int) -> dict[str, Any]: + if k not in KS: + raise ValueError("k must be 5, 10, 20 or 32") + if any(key in row for key in ("positive_ids", "target_id", "no_positive", "label")): + raise ValueError("selector input must be label-free retrieval output") + fingerprint = row.get("input_sha256") + if not isinstance(fingerprint, str) or re.fullmatch(r"[0-9a-f]{64}", fingerprint) is None: + raise ValueError("retrieval input requires a valid input_sha256 before inference") + candidates = row["candidates"] + ids = [item["id"] for item in candidates] + if any(not isinstance(key, str) or not key for key in ids) or len(ids) != len(set(ids)): + raise ValueError("candidate IDs must be unique nonempty strings") + if ids != row["generated_candidates"]: + raise ValueError("candidate order disagrees with retriever output") + selected = candidates[:k] + if any(not isinstance(item["text"], str) for item in selected): + raise ValueError("candidate text must be a string") + return {"state": canonical(row["state"]), "question": QUESTION, + "candidates": [{"id": item["id"], "text": item["text"]} for item in selected]} + + +def decode(scores: list[float], answerability_logit: float, + ids: list[str], threshold: float = 0.55) -> dict[str, Any]: + if len(scores) != len(ids) or not ids or not all(math.isfinite(x) for x in scores): + raise ModelOutputError("model returned invalid scores") + if not math.isfinite(answerability_logit): + raise ModelOutputError("model returned non-finite answerability") + pivot = max(scores) + weights = [math.exp(value - pivot) for value in scores] + total = sum(weights) + probabilities = [value / total for value in weights] + z = math.exp(-abs(answerability_logit)) + answerability = 1 / (1 + z) if answerability_logit >= 0 else z / (1 + z) + index = max(range(len(ids)), key=probabilities.__getitem__) + confidence = probabilities[index] * answerability + reason = ("answerability_below_threshold" if answerability < 0.5 else + "confidence_below_threshold" if confidence < threshold else None) + return {"selection": ids[index] if reason is None else None, + "status": "ok" if reason is None else "abstain", "reason": reason, + "probabilities": dict(zip(ids, probabilities)), "confidence": confidence, + "answerability": answerability, "raw_scores": scores, + "answerability_logit": answerability_logit} + + +class DirectSelector: + """Offline, verified shared pilot graph; no fitted calibration is implied.""" + + def __init__(self, manifest: Path, expected_digest: str) -> None: + import numpy as np + import onnxruntime as ort + from tokenizers import Tokenizer + + spec = importlib.util.spec_from_file_location("shared_export", Path(__file__).with_name("export_shared_bundle.py")) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.verify_manifest(manifest, expected_digest=expected_digest) + self.np = np + self.manifest = manifest + self.tokenizer = Tokenizer.from_file(str(manifest.parent / "tokenizer/tokenizer.json")) + self.tokenizer.no_truncation() + self.tokenizer.no_padding() + settings = ort.SessionOptions() + settings.intra_op_num_threads = 1 + settings.inter_op_num_threads = 1 + self.encoder = ort.InferenceSession(str(manifest.parent / "encoder.onnx"), settings, + providers=["CPUExecutionProvider"]) + self.head = ort.InferenceSession(str(manifest.parent / "direct.onnx"), settings, + providers=["CPUExecutionProvider"]) + self.provenance = {"backend": "direct", "model_manifest_sha256": expected_digest, + "tokenizer_sha256": file_hash(manifest.parent / "tokenizer/tokenizer.json"), + "ort_version": ort.__version__, "provider": "CPUExecutionProvider", + "calibration": "none; historical default thresholds 0.55/0.5", + "threads": 1, "sequence_padding": "longest candidate input", + "candidate_padding": "live", "input_budget_tokens": 512} + + def __call__(self, request: dict[str, Any]) -> dict[str, Any]: + np = self.np + candidates = request["candidates"] + if not 2 <= len(candidates) <= 32: + raise ValueError("candidate_count_outside_2_32") + prefix = f'{request["state"]}\nQuestion: {request["question"]}' + all_text = prefix + "\nCandidates:\n" + "\n".join(item["text"] for item in candidates) + token_count = len(self.tokenizer.encode(all_text).ids) + if token_count > 512: + return {"selection": None, "status": "error", "reason": "input_overflow", + "question_tokens": token_count} + encoded = [self.tokenizer.encode(prefix + "\nCandidate: " + item["text"]) + for item in candidates] + length = max(len(item.ids) for item in encoded) + if length > 512: + return {"selection": None, "status": "error", "reason": "candidate_input_overflow", + "question_tokens": token_count} + ids = np.full((1, len(candidates), length), self.tokenizer.token_to_id("[PAD]"), dtype=np.int64) + masks = np.zeros_like(ids) + types = np.zeros_like(ids) + for index, item in enumerate(encoded): + ids[0, index, :len(item.ids)] = item.ids + masks[0, index, :len(item.ids)] = item.attention_mask + types[0, index, :len(item.ids)] = item.type_ids + option_mask = np.ones((1, len(candidates)), dtype=np.bool_) + feed = {"input_ids": ids, "attention_mask": masks, "token_type_ids": types, + "option_mask": option_mask} + embeddings, pooled = self.encoder.run(None, feed) + scores, answerability = self.head.run(None, {"candidate_embeddings": embeddings, + "pooled": pooled, "option_mask": option_mask}) + result = decode(scores[0].tolist(), float(answerability[0]), + [item["id"] for item in candidates]) + result.update(question_tokens=token_count, + input_array_sha256=object_hash({key: value.tolist() for key, value in feed.items()})) + return result + + +def predict_file(source: Path, output: Path, *, split: str, k: int, + select: Callable[[dict[str, Any]], dict[str, Any]], + provenance: dict[str, Any]) -> dict[str, Any]: + if output.exists() or output.with_suffix(output.suffix + ".partial").exists(): + raise ValueError("refusing to overwrite prediction evidence") + output.parent.mkdir(parents=True, exist_ok=True) + partial = output.with_suffix(output.suffix + ".partial") + seen: set[str] = set() + counts: Counter[str] = Counter() + source_digest = file_hash(source) + config = {"split": split, "k": k, "provenance": provenance, "adapted": False, + "question": QUESTION, "input_overflow": "preserve error; no truncation"} + config_digest = object_hash(config) + with source.open() as handle, partial.open("x") as destination: + for line in handle: + row = json.loads(line) + if row["split"] != split: + continue + if row["id"] in seen: + raise ValueError("duplicate retrieval row ID") + seen.add(row["id"]) + request = make_request(row, k) + started = time.perf_counter() + try: + result = select(request) + except ModelOutputError: + raise + except Exception as exc: # noqa: BLE001 -- retain per-row provider failures in the denominator + result = {"selection": None, "status": "error", "reason": type(exc).__name__, + "error": str(exc)} + elapsed = (time.perf_counter() - started) * 1000 + record = {"id": row["id"], "task_id": row["task_id"], "split": split, + "k": k, "generated_candidates": [item["id"] for item in request["candidates"]], + "request_sha256": object_hash(request), "config_sha256": config_digest, + "retrieval_input_sha256": row["input_sha256"], "elapsed_ms": elapsed, + **result} + if record["selection"] is not None and record["selection"] not in record["generated_candidates"]: + raise ValueError("selector returned an unknown candidate") + counts[record["status"]] += 1 + destination.write(canonical(record) + "\n") + destination.flush() + if not seen: + raise ValueError("no rows matched requested split") + partial.rename(output) + report = {"schema": "vons.mind2web-prediction/v1", "config": config, + "config_sha256": config_digest, "source_sha256": source_digest, + "predictions_sha256": file_hash(output), "runner_sha256": file_hash(Path(__file__)), + "rows": len(seen), "counts": dict(counts), "selection_reused_across_k": False, + "timing_scope": "tokenization/inference/decoding, not a controlled benchmark", + "scope": "action target selection; not browser task success"} + output.with_suffix(".manifest.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--retriever", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--split", choices=["test_task", "test_website", "test_domain"], required=True) + parser.add_argument("--k", type=int, choices=KS, required=True) + parser.add_argument("--backend", choices=["direct", "lexical_top1"], default="direct") + parser.add_argument("--manifest", type=Path) + parser.add_argument("--manifest-sha256") + args = parser.parse_args() + if args.backend == "direct": + if args.manifest is None or not args.manifest_sha256: + parser.error("Direct requires a shared manifest and externally pinned --manifest-sha256") + select = DirectSelector(args.manifest, args.manifest_sha256) + provenance = select.provenance + else: + def select(request: dict[str, Any]) -> dict[str, Any]: + candidates = request["candidates"] + return {"selection": candidates[0]["id"] if candidates else None, + "status": "ok" if candidates else "abstain", "reason": None} + provenance = {"backend": "lexical_top1", "input_budget": "unrestricted retrieval baseline"} + report = predict_file(args.retriever, args.output, split=args.split, k=args.k, + select=select, provenance=provenance) + print(json.dumps({"rows": report["rows"], "counts": report["counts"]})) + + +if __name__ == "__main__": + main() diff --git a/tools/prepare_public_release.py b/tools/prepare_public_release.py new file mode 100644 index 0000000000000000000000000000000000000000..f5794e54ec16e7da737e33623b326e47243e68be --- /dev/null +++ b/tools/prepare_public_release.py @@ -0,0 +1,152 @@ +"""Audit and export an explicit public source set without uploading anything. + +Git ignore rules protect ordinary staging; this separate allowlist also protects +folder-based uploads. Pattern scanning is a heuristic, not a rights review or a +complete secret detector. Findings report paths and categories, never matches. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import tempfile +from pathlib import Path, PurePosixPath + +DENIED_PARTS = { + ".git", ".git_backup", ".venv", "node_modules", "__pycache__", "data", + "artifacts", "reports", "research", "private", "restricted", "models", + "weights", "checkpoints", "releases", ".aws", ".ssh", ".cache", +} +TEXT_SUFFIXES = {".py", ".ts", ".mjs", ".md", ".json", ".toml", ".yml", ".cff", ".html"} +TEXT_NAMES = {".gitignore", ".gitattributes", "LICENSE", "NOTICE"} +PATTERNS = { + "private-key": re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), + "provider-token": re.compile( + r"\b(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,}" + r"|hf_[A-Za-z0-9]{25,}|sk-(?:proj-)?[A-Za-z0-9_-]{30,})\b" + ), + "aws-access-key": re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"), + "personal-home-path": re.compile(r"/(?:Users|home)/[A-Za-z0-9_.-]+/"), + "private-writing-session": re.compile( + r"localhost[:]8766|/frames/[0-9a-f-]{36}|proj_[0-9a-f]{12}" + ), + "credential-url": re.compile(r"https?://[^\s/:@]+:[^\s/@]+@"), +} + + +def git_files(root: Path) -> set[str]: + result = subprocess.run( + ["git", "-c", "core.quotepath=false", "ls-files", "-z", "--cached", "--others", + "--exclude-standard"], cwd=root, check=True, capture_output=True, + ) + return {p for p in result.stdout.decode().split("\0") if p} + + +def inspect_file(root: Path, name: str, reviewed_binary: str | None = None) -> bytes: + relative = PurePosixPath(name) + if (relative.is_absolute() or ".." in relative.parts or "\\" in name + or relative.as_posix() != name or not relative.parts + or DENIED_PARTS.intersection(relative.parts) + or any(p.startswith(".env") for p in relative.parts)): + raise ValueError(f"disallowed path: {name}") + source = root / name + if any((root / Path(*relative.parts[:i])).is_symlink() + for i in range(1, len(relative.parts) + 1)): + raise ValueError(f"symlink is not public source: {name}") + if not source.is_file() or not source.resolve().is_relative_to(root.resolve()): + raise ValueError(f"missing or external source: {name}") + if source.stat().st_size > 20 * 1024 * 1024: + raise ValueError(f"oversized source: {name}") + content = source.read_bytes() + binary_location = ( + relative.suffix == ".pdf" and relative.parts[:2] == ("docs", "publications") + ) or (relative.suffix == ".png" and relative.parts[:2] == ("docs", "assets")) + if binary_location: + signature = b"%PDF-" if relative.suffix == ".pdf" else b"\x89PNG\r\n\x1a\n" + if (not content.startswith(signature) or not reviewed_binary + or hashlib.sha256(content).hexdigest() != reviewed_binary): + raise ValueError(f"binary needs an exact reviewed digest: {name}") + return content + if relative.suffix not in TEXT_SUFFIXES and relative.name not in TEXT_NAMES: + raise ValueError(f"unreviewed file type: {name}") + try: + text = content.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError(f"non-text source: {name}") from exc + if "\0" in text: + raise ValueError(f"binary content in text file: {name}") + for category, pattern in PATTERNS.items(): + if pattern.search(text): + raise ValueError(f"{category}: {name}") + return content + + +def audit(root: Path) -> tuple[dict, dict[str, bytes]]: + spec = json.loads((root / "configs/public-release.json").read_text(encoding="utf-8")) + names = spec["files"] + if (not isinstance(names, list) or not names + or not all(isinstance(n, str) for n in names) or len(set(names)) != len(names)): + raise ValueError("release files must be a nonempty unique path list") + candidates = git_files(root) + if candidates != set(names): + raise ValueError(json.dumps({"unexpected_public_paths": sorted(candidates - set(names)), + "missing_or_ignored_paths": sorted(set(names) - candidates)})) + content = {name: inspect_file(root, name, spec.get("reviewed_binaries", {}).get(name)) + for name in sorted(names)} + return spec, content + + +def prepare(root: Path, output: Path | None = None, target: str = "github") -> dict: + spec, content = audit(root) + if target not in {"github", "huggingface"}: + raise ValueError("unknown release target") + if target == "huggingface": + content["README.md"] = content["docs/HUGGING_FACE.md"].replace(b"](../", b"](") + manifest = { + "schema": "vons.public-source/v1", "target": target, + "status": "prepared_local_only", "license_status": spec["license_status"], + "weights_included": False, "datasets_included": False, + "files": [{"path": name, "bytes": len(value), + "sha256": hashlib.sha256(value).hexdigest()} + for name, value in sorted(content.items())], + } + if output is not None: + output = output.resolve() + if output.exists(): + raise ValueError("refusing to overwrite a release directory") + if output == root.resolve() or output in root.resolve().parents: + raise ValueError("output must not replace the source tree") + output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=output.parent) as temporary: + stage = Path(temporary) / "release" + stage.mkdir() + for name, value in content.items(): + destination = stage / name + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(value) + (stage / "RELEASE_MANIFEST.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + stage.rename(output) + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--target", choices=["github", "huggingface"], default="github") + parser.add_argument("--output", type=Path, help="Create a new local export directory") + args = parser.parse_args() + try: + result = prepare(args.root.resolve(), args.output, args.target) + except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as exc: + parser.exit(1, f"Public release audit failed: {exc}\n") + print(json.dumps({"status": result["status"], "files": len(result["files"]), + "bytes": sum(f["bytes"] for f in result["files"]), + "license_status": result["license_status"], "target": args.target}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/recover_historical_manifest.py b/tools/recover_historical_manifest.py new file mode 100644 index 0000000000000000000000000000000000000000..7aea4bde8144121a8585eacf6446b02788b1a5b7 --- /dev/null +++ b/tools/recover_historical_manifest.py @@ -0,0 +1,105 @@ +"""Recover an exact historical bundle manifest from a Codex JSONL session log. + +This is a provenance utility for preserving a manifest whose digest is already +embedded in a raw browser benchmark. It does not change the current bundle +manifest and refuses to write unless the recovered bytes match the expected +digest. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def recover( + session_log: Path, + *, + ordinal: int, + source_hash_in_log: str, + replacement_source_hash: str, + expected_manifest_hash: str, + output: Path, + bundle_root: Path | None = None, +) -> dict[str, Any]: + """Recover one command output and validate its exact historical digest.""" + + stdout: str | None = None + with session_log.open(encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + record = json.loads(line) + if record.get("ordinal") != ordinal: + continue + payload = record.get("payload", {}) + item = payload.get("item", {}) if isinstance(payload, dict) else {} + candidate = item.get("stdout") if isinstance(item, dict) else None + if isinstance(candidate, str): + stdout = candidate + break + raise ValueError(f"ordinal {ordinal} has no command stdout (line {line_number})") + if stdout is None: + raise ValueError(f"ordinal {ordinal} was not found in {session_log}") + + source_count = stdout.count(source_hash_in_log) + if source_count != 1: + raise ValueError(f"expected one source hash in the session output, found {source_count}") + raw = stdout.replace(source_hash_in_log, replacement_source_hash, 1).encode("utf-8") + payload = json.loads(raw.decode("utf-8")) + source_snapshot = payload.get("source", {}).get("source_snapshot", {}) + if source_snapshot.get("sha256") != replacement_source_hash: + raise ValueError("historical manifest does not contain the replacement source hash") + + # The session output contains the pre-change source hash. The caller passes + # the hash recorded in the browser run separately through the same option. + actual_hash = _sha256_bytes(raw) + if actual_hash != expected_manifest_hash: + raise ValueError(f"recovered manifest hash {actual_hash} != expected {expected_manifest_hash}") + if payload.get("summary", {}).get("manifest_bytes") != len(raw): + raise ValueError("historical manifest summary does not match recovered byte count") + bundle_files_match: bool | None = None + if bundle_root is not None: + bundle_files_match = True + for item in payload.get("files", []): + path = bundle_root / str(item["path"]) + if not path.is_file() or path.stat().st_size != item.get("bytes"): + bundle_files_match = False + break + if _sha256_bytes(path.read_bytes()) != item.get("sha256"): + bundle_files_match = False + break + if not bundle_files_match: + raise ValueError("historical manifest file inventory does not match the supplied bundle") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(raw) + return { + "output": str(output), + "bytes": len(raw), + "sha256": actual_hash, + "source_snapshot_sha256": replacement_source_hash, + "session_ordinal": ordinal, + "bundle_files_match": bundle_files_match, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--session-log", type=Path, required=True) + parser.add_argument("--ordinal", type=int, required=True) + parser.add_argument("--source-hash-in-log", required=True) + parser.add_argument("--replacement-source-hash", required=True) + parser.add_argument("--expected-manifest-hash", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--bundle-root", type=Path) + args = parser.parse_args() + print(json.dumps(recover(**vars(args)), indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tools/replay_mind2web_evaluation.py b/tools/replay_mind2web_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..81c8deb7fa85f9e45017c40c86c0c239f9ec79fc --- /dev/null +++ b/tools/replay_mind2web_evaluation.py @@ -0,0 +1,506 @@ +"""Replay frozen Mind2Web predictions without running a model or changing inputs. + +Requires -k.jsonl and matching .manifest.json sidecars. By default the +run index must cover all three official held-out splits and k=5/10/20/32. +--allow-subset permits an explicitly partial run index, never unknown cells. +Missing prediction records remain in the gold denominator and are reported +separately from recorded errors and abstentions. Existing output is preserved. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import importlib.util +import json +import re +import tempfile +from collections import Counter +from collections.abc import Iterator, Mapping +from pathlib import Path +from types import ModuleType +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vons.mind2web import Mind2WebExample + +SPLITS = ("test_task", "test_website", "test_domain") +KS = (5, 10, 20, 32) +CELLS = tuple((split, k) for split in SPLITS for k in KS) +SOURCE_FILES = ( + "vons/mind2web.py", + "tools/evaluate_mind2web.py", + "tools/predict_mind2web.py", + "tools/replay_mind2web_evaluation.py", +) +SCOPE = ( + "Replay of saved exact-k action-target predictions over the released candidate pool; " + "no model execution, test-set adaptation, or browser task success." +) + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict): + raise TypeError(f"{label} must be a JSON object") + return value + + +def _nonfinite(value: str) -> None: + raise ValueError("non-finite JSON number") + + +def _load(path: Path) -> dict[str, object]: + return _object( + json.loads(path.read_text(encoding="utf-8"), parse_constant=_nonfinite), "JSON document" + ) + + +def _jsonl(path: Path) -> Iterator[dict[str, object]]: + with path.open(encoding="utf-8") as handle: + for line in handle: + if line.strip(): + yield _object(json.loads(line, parse_constant=_nonfinite), "JSONL row") + + +def _hash(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _object_hash(value: object) -> str: + payload = json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _digest(value: object, label: str) -> str: + if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None: + raise ValueError(f"{label} must be a SHA-256 digest") + return value + + +def _integer(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{label} must be a nonnegative integer") + return value + + +def _counts(value: object) -> dict[str, int]: + counts = _object(value, "status counts") + return { + key: count for key, raw in counts.items() if (count := _integer(raw, "status count")) > 0 + } + + +def _source_paths() -> dict[str, Path]: + root = Path(__file__).resolve().parents[1] + return {name: root / name for name in SOURCE_FILES} + + +def _unchanged(snapshot: Mapping[Path, str]) -> None: + for path, digest in snapshot.items(): + if not path.is_file() or _hash(path) != digest: + raise ValueError("source mutation during replay") + + +def _load_evaluator(path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location("vons_frozen_mind2web_evaluator", path) + if spec is None or spec.loader is None: + raise RuntimeError("could not load the frozen Mind2Web evaluator") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _read_index( + index: Mapping[str, object], + allow_subset: bool, +) -> dict[tuple[str, int], dict[str, object]]: + runs = index.get("runs") + if not isinstance(runs, list) or not runs: + raise ValueError("run index must contain a nonempty runs array") + cells: dict[tuple[str, int], dict[str, object]] = {} + for raw in runs: + run = _object(raw, "run") + split = run.get("split") + k = _integer(run.get("k"), "run k") + if split not in SPLITS or k not in KS: + raise ValueError("run index contains an unknown official split/k cell") + cell = (str(split), k) + if cell in cells: + raise ValueError("run index contains a duplicate split/k cell") + _integer(run.get("rows"), "run rows") + _digest(run.get("predictions_sha256"), "run prediction hash") + _counts(run.get("counts")) + cells[cell] = run + if not allow_subset and set(cells) != set(CELLS): + raise ValueError( + "all 12 official split/k cells are required; use --allow-subset explicitly" + ) + _digest(index.get("source_sha256"), "retrieval source hash") + _digest(index.get("runner_sha256"), "prediction runner hash") + return cells + + +def _read_gold(rows_path: Path, evaluator: ModuleType) -> dict[str, dict[str, Mind2WebExample]]: + by_split: dict[str, dict[str, Mind2WebExample]] = {split: {} for split in SPLITS} + seen: set[str] = set() + tasks: dict[str, str] = {} + for raw in _jsonl(rows_path): + row = evaluator.Mind2WebExample.from_mapping(raw) + if row.split not in SPLITS or row.task_id is None: + raise ValueError("gold rows require an official split and task_id") + if row.example_id in seen: + raise ValueError("duplicate gold row id") + seen.add(row.example_id) + if row.task_id in tasks and tasks[row.task_id] != row.split: + raise ValueError("gold task identity appears in multiple splits") + tasks[row.task_id] = row.split + by_split[row.split][row.example_id] = row + return by_split + + +def _inspect_predictions( + path: Path, + *, + split: str, + k: int, + config_hash: str, + gold: Mapping[str, Mind2WebExample], +) -> dict[str, object]: + seen: set[str] = set() + statuses: Counter[str] = Counter() + reasons: Counter[str | None] = Counter() + status_reasons: Counter[tuple[str, str | None]] = Counter() + failure_reasons: Counter[str] = Counter() + missing_reason_fields = 0 + for row in _jsonl(path): + row_id = row.get("id") + if not isinstance(row_id, str) or row_id not in gold or row_id in seen: + raise ValueError("prediction id is unknown or duplicated") + seen.add(row_id) + if ( + row.get("split") != split + or _integer(row.get("k"), "prediction k") != k + or row.get("task_id") != gold[row_id].task_id + ): + raise ValueError("prediction split/k/task identity disagrees with its cell") + if row.get("config_sha256") != config_hash: + raise ValueError("prediction config hash disagrees with its manifest") + for key in ("request_sha256", "retrieval_input_sha256"): + _digest(row.get(key), key) + candidates = row.get("generated_candidates") + if ( + not isinstance(candidates, list) + or len(candidates) > k + or any(not isinstance(item, str) or not item for item in candidates) + or len(candidates) != len(set(candidates)) + or not set(candidates).issubset(gold[row_id].candidate_ids) + ): + raise ValueError("prediction candidates violate the gold universe or exact-k limit") + status = row.get("status") + selection = row.get("selection") + if status not in ("ok", "error", "abstain"): + raise ValueError("unknown prediction status") + if (status == "ok" and (not isinstance(selection, str) or selection not in candidates)) or ( + status != "ok" and selection is not None + ): + raise ValueError("prediction status disagrees with its selection") + reason = row.get("reason") + if reason is not None and (not isinstance(reason, str) or not reason): + raise ValueError("prediction reason must be null or a nonempty string") + missing_reason_fields += int("reason" not in row) + statuses[status] += 1 + reasons[reason] += 1 + status_reasons[(status, reason)] += 1 + if status != "ok" and reason is not None: + failure_reasons[reason] += 1 + denominator = len(gold) + outcomes = { + "accepted": statuses["ok"], + "error": statuses["error"], + "abstain": statuses["abstain"], + "missing": denominator - len(seen), + } + + def reason_order(value: str | None) -> tuple[bool, str]: + return value is not None, value or "" + + return { + "prediction_rows": len(seen), + "execution_status_counts": dict(sorted(statuses.items())), + "execution_reason_counts": [ + {"reason": reason, "count": reasons[reason]} + for reason in sorted(reasons, key=reason_order) + ], + "execution_status_reason_counts": [ + {"status": status, "reason": reason, "count": count} + for (status, reason), count in sorted( + status_reasons.items(), key=lambda item: (item[0][0], reason_order(item[0][1])) + ) + ], + "execution_failure_counts": dict(sorted(failure_reasons.items())), + "missing_reason_fields": missing_reason_fields, + "execution_outcomes": outcomes, + "acceptance_coverage": statuses["ok"] / denominator if denominator else None, + "prediction_coverage": len(seen) / denominator if denominator else None, + "coverage_denominator_rows": denominator, + "missing_prediction_record_count": outcomes["missing"], + "missing_prediction_interpretation": ( + "Missing records have no raw status; error and abstain are observed records. " + "The frozen metric's missing_predictions also includes absent selections on recalled rows." + ), + } + + +def _write(path: Path, value: object) -> None: + path.write_text( + json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + + +def _write_table(path: Path, runs: list[dict[str, object]]) -> None: + fields = ( + "split", + "k", + "rows", + "positive_rows", + "recalled_rows", + "recall_micro", + "recall_task_macro", + "taskmacroCIlo", + "taskmacroCIhi", + "conditional_accuracy_micro", + "conditional_accuracy_task_macro", + "coverage", + "error_count", + "abstain_count", + "missing_count", + ) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fields) + writer.writeheader() + for run in runs: + metrics = _object(run["metrics"], "metrics") + bootstrap = _object(run["bootstrap"], "bootstrap") + outcomes = _object(run["execution_outcomes"], "execution outcomes") + interval = bootstrap["candidate_recall_task_macro_ci95"] + if interval is not None and (not isinstance(interval, list) or len(interval) != 2): + raise ValueError("frozen evaluator returned an invalid bootstrap interval") + writer.writerow( + { + "split": run["split"], + "k": run["k"], + "rows": metrics["rows"], + "positive_rows": metrics["positive_rows"], + "recalled_rows": metrics["recalled_positive_rows"], + "recall_micro": metrics["candidate_recall_step_micro"], + "recall_task_macro": metrics["candidate_recall_task_macro"], + "taskmacroCIlo": interval[0] if interval is not None else None, + "taskmacroCIhi": interval[1] if interval is not None else None, + "conditional_accuracy_micro": metrics[ + "selection_accuracy_given_recall_step_micro" + ], + "conditional_accuracy_task_macro": metrics[ + "selection_accuracy_given_recall_task_macro" + ], + "coverage": run["acceptance_coverage"], + "error_count": outcomes["error"], + "abstain_count": outcomes["abstain"], + "missing_count": outcomes["missing"], + } + ) + + +def replay( + rows_path: Path, + run_index_path: Path, + predictions_dir: Path, + output_dir: Path, + *, + allow_subset: bool = False, +) -> dict[str, object]: + if output_dir.exists(): + raise ValueError("refusing to overwrite replay evidence") + sources = _source_paths() + source_hashes = {name: _hash(path) for name, path in sources.items()} + code_snapshot = {path: source_hashes[name] for name, path in sources.items()} + snapshot = {**code_snapshot, rows_path: _hash(rows_path), run_index_path: _hash(run_index_path)} + index = _load(run_index_path) + cells = _read_index(index, allow_subset) + if index["runner_sha256"] != source_hashes["tools/predict_mind2web.py"]: + raise ValueError("prediction runner source differs from the frozen run index") + evaluator = _load_evaluator(sources["tools/evaluate_mind2web.py"]) + _unchanged(code_snapshot) + gold = _read_gold(rows_path, evaluator) + common_config_hash: str | None = None + prepared: dict[tuple[str, int], tuple[Path, Path, dict[str, object], dict[str, object]]] = {} + for split, k in CELLS: + if (split, k) not in cells: + continue + run = cells[(split, k)] + if not gold[split]: + raise ValueError("requested split has no gold rows") + path = predictions_dir / f"{split}-k{k}.jsonl" + manifest_path = path.with_suffix(".manifest.json") + snapshot[path] = _hash(path) + snapshot[manifest_path] = _hash(manifest_path) + manifest = _load(manifest_path) + if manifest.get("schema") != "vons.mind2web-prediction/v1": + raise ValueError("unsupported prediction manifest schema") + for record in (run, manifest): + if record.get("predictions_sha256") != snapshot[path]: + raise ValueError("prediction file hash disagrees with run index or manifest") + if ( + manifest.get("source_sha256") != index["source_sha256"] + or manifest.get("runner_sha256") != index["runner_sha256"] + or manifest.get("selection_reused_across_k") is not False + ): + raise ValueError("prediction provenance or exact-k contract mismatch") + config = _object(manifest.get("config"), "prediction config") + if config.get("split") != split or _integer(config.get("k"), "config k") != k: + raise ValueError("prediction manifest config split/k mismatch") + config_hash = _object_hash(config) + if manifest.get("config_sha256") != config_hash: + raise ValueError("prediction manifest config hash mismatch") + if "config_sha256" in run and run["config_sha256"] != config_hash: + raise ValueError("run index config hash mismatch") + shared = _object_hash( + {key: value for key, value in config.items() if key not in {"split", "k"}} + ) + if common_config_hash is not None and shared != common_config_hash: + raise ValueError("prediction cells mix incompatible frozen configurations") + common_config_hash = shared + inspection = _inspect_predictions( + path, split=split, k=k, config_hash=config_hash, gold=gold[split] + ) + for record in (run, manifest): + if _integer(record.get("rows"), "prediction rows") != inspection["prediction_rows"]: + raise ValueError("prediction row count disagrees with run index or manifest") + if _counts(record.get("counts")) != inspection["execution_status_counts"]: + raise ValueError("raw status counts disagree with run index or manifest") + prepared[(split, k)] = (path, manifest_path, manifest, inspection) + _unchanged(snapshot) + output_dir.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".mind2web-replay-", dir=output_dir.parent + ) as temporary: + staging = Path(temporary) / "reports" + staging.mkdir() + runs: list[dict[str, object]] = [] + for (split, k), (path, manifest_path, manifest, inspection) in prepared.items(): + _unchanged(code_snapshot) + report = evaluator.evaluate_files(rows_path, path, [k], split=split) + _unchanged({**code_snapshot, rows_path: snapshot[rows_path], path: snapshot[path]}) + if ( + report["rows"] != len(gold[split]) + or report["prediction_rows"] != inspection["prediction_rows"] + or report["input_sha256"] + != {"rows": snapshot[rows_path], "predictions": snapshot[path]} + ): + raise ValueError("frozen evaluator input/count provenance mismatch") + report.update(inspection) + report["source_sha256"] = source_hashes + report["replay_provenance"] = { + "run_index_sha256": snapshot[run_index_path], + "prediction_manifest_sha256": snapshot[manifest_path], + "retrieval_source_sha256": index["source_sha256"], + "config_sha256": manifest["config_sha256"], + } + name = f"{split}-k{k}-metrics.json" + _write(staging / name, report) + runs.append( + { + "split": split, + "k": k, + "report": str(output_dir / name), + "sha256": _hash(staging / name), + "predictions_sha256": snapshot[path], + "prediction_manifest_sha256": snapshot[manifest_path], + "config_sha256": manifest["config_sha256"], + "counts": inspection["execution_status_counts"], + **inspection, + "metrics": report["metrics_by_k"][0], + "bootstrap": report["task_bootstrap_by_k"][str(k)], + } + ) + table_path = staging / "summary.csv" + _write_table(table_path, runs) + summary: dict[str, object] = { + "schema": "vons.mind2web-run-summary/v1", + "scope": SCOPE, + "source_sha256": source_hashes, + "input_sha256": { + "rows": snapshot[rows_path], + "run_index": snapshot[run_index_path], + "retrieval_source_recorded": index["source_sha256"], + }, + "prediction_runner_sha256": index["runner_sha256"], + "common_config_sha256": common_config_hash, + "matrix": { + "complete": set(cells) == set(CELLS), + "allow_subset": allow_subset, + "expected_cells": [{"split": split, "k": k} for split, k in CELLS], + "included_cells": [{"split": run["split"], "k": run["k"]} for run in runs], + }, + "runs": runs, + "table": { + "path": str(output_dir / table_path.name), + "sha256": _hash(table_path), + "bytes": table_path.stat().st_size, + "null_encoding": "empty CSV cell; null in JSON", + "coverage": "accepted raw status=ok records divided by all gold rows in the split", + "missing_count": "gold rows with no prediction record, excluding error/abstain records", + "taskmacroCI": "95% task-bootstrap interval for candidate recall task macro", + }, + "limitations": [ + "Retrieval-source and per-request hashes are preserved from predictions; this CLI does not reload retrieval text or model assets.", + "Missing prediction records remain in the gold denominator; bootstrap settings come unchanged from the frozen evaluator.", + ], + } + _write(staging / "summary.json", summary) + _unchanged(snapshot) + if output_dir.exists(): + raise ValueError("refusing to overwrite replay evidence") + staging.rename(output_dir) + return summary + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=Path, required=True) + parser.add_argument("--run-index", type=Path, required=True) + parser.add_argument("--predictions-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--allow-subset", action="store_true") + args = parser.parse_args() + try: + summary = replay( + args.rows, + args.run_index, + args.predictions_dir, + args.output_dir, + allow_subset=args.allow_subset, + ) + except (ValueError, TypeError, OSError, RuntimeError) as exc: + parser.exit( + 2, f"Mind2Web replay failed ({type(exc).__name__}); no replay output committed.\n" + ) + print( + json.dumps( + { + "status": "replayed", + "cells": len(summary["runs"]), + "complete_matrix": summary["matrix"]["complete"], + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tools/summarize_browser_benchmark.py b/tools/summarize_browser_benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..e9da0cfbe74d1bf260d37033763df70fe234450a --- /dev/null +++ b/tools/summarize_browser_benchmark.py @@ -0,0 +1,748 @@ +"""Summarize a JSONL stream of Vons browser benchmark samples. + +Inputs are one raw browser sample per JSONL line. Each line must carry the +full per-sample schema plus a provenance record that describes the session +configuration. The tool performs strict validation before producing any +summary: identity duplicates, malformed types, non-finite timings, mixed +experimental conditions, and diffusion rows without noise hashes all cause +the tool to fail before the output path is touched. + +Successful rows contribute to linearly interpolated p50/p95 percentiles per +timing stage. Failed rows are counted explicitly and retain nullable timing +slots. Heap maxima are derived only from actually-measured observations. +Measurement completeness (>= 5 independent session starts, >= 10 warmups +excluded per session, >= 100 timed rows total) is reported as a separate +flag without elevating partial runs to "complete". Because the page harness +performs in-page reloads only, "process cold" is never claimed, and the +requested provider is never represented as proven. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +TIMING_FIELDS: tuple[str, ...] = ( + "tokenizeMs", + "prepareFeedMs", + "inferenceAndReadbackMs", + "postprocessMs", + "totalRequestMs", +) + +HEAP_FIELDS: tuple[str, ...] = ( + "js_heap_used_bytes_before", + "js_heap_used_bytes_after", +) + +PROVIDERS = {"wasm", "webgpu"} +BACKENDS = {"direct", "diffusion"} +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") + +REQUIRED_IDENTITY_FIELDS: tuple[str, ...] = ("run_id", "session_start", "repeat") +REQUIRED_TOP_FIELDS: tuple[str, ...] = ( + "run_id", + "session_start", + "repeat", + "case_id", + "backend", + "requested_provider", + "observed_provider", + "bundle_hash", + "tokenizer_hash", + "input_hash", + "candidate_count", + "padded_option_slots", + "sequence_length", + "seed", + "ort_version", + "status", +) + +REQUIRED_PROVENANCE_FIELDS: tuple[str, ...] = ( + "environment", + "execution_provider_evidence", + "warmup_excluded_per_session", + "session_start_semantics", +) + +CONDITION_KEYS: tuple[str, ...] = ( + "run_id", + "backend", + "requested_provider", + "observed_provider", + "bundle_hash", + "tokenizer_hash", + "candidate_count", + "padded_option_slots", + "sequence_length", + "ort_version", +) + +MAX_PADDED_OPTION_SLOTS = 32 +MAX_SEQUENCE_LENGTH = 512 + + +def _finite_number( + value: Any, label: str, *, allow_none: bool = False +) -> float | None: + if value is None and allow_none: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{label} must be a finite number or null") + numeric = float(value) + if not math.isfinite(numeric) or numeric < 0: + raise ValueError(f"{label} must be a finite non-negative number") + return numeric + + +def _positive_integer(value: Any, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{label} must be a positive integer") + return value + + +def _nonnegative_integer(value: Any, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{label} must be a non-negative integer") + return value + + +def _sha256(value: Any, label: str, *, allow_none: bool = False) -> str | None: + if value is None and allow_none: + return None + if not isinstance(value, str) or SHA256_PATTERN.fullmatch(value) is None: + raise ValueError(f"{label} must be a lowercase SHA-256 hex string") + return value + + +def _nonempty_string(value: Any, label: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{label} must be a non-empty string") + return value + + +def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + if not 0.0 <= fraction <= 1.0: + raise ValueError("percentile fraction must be within [0, 1]") + ordered = sorted(values) + index = (len(ordered) - 1) * fraction + lower = math.floor(index) + upper = math.ceil(index) + if lower == upper: + return float(ordered[lower]) + return float( + ordered[lower] + (ordered[upper] - ordered[lower]) * (index - lower) + ) + + +def _provenance_condition_signature(provenance: dict[str, Any]) -> dict[str, Any]: + environment = provenance["environment"] + evidence = provenance["execution_provider_evidence"] + warmup = provenance["warmup_excluded_per_session"] + semantics = provenance["session_start_semantics"] + return { + "environment": environment, + "execution_provider_evidence": evidence, + "warmup_excluded_per_session": warmup, + "session_start_semantics": semantics, + } + + +def _validate_row(row: dict[str, Any], line_no: int) -> None: + if not isinstance(row, dict): + raise TypeError(f"line {line_no}: sample must be a JSON object") + provenance = row.get("provenance") + if not isinstance(provenance, dict): + raise TypeError(f"line {line_no}: provenance object is required") + for field in REQUIRED_TOP_FIELDS: + if field not in row: + raise ValueError(f"line {line_no}: missing required field {field}") + for field in REQUIRED_PROVENANCE_FIELDS: + if field not in provenance: + raise ValueError( + f"line {line_no}: provenance missing required field {field}" + ) + + _nonempty_string(row["run_id"], f"line {line_no}.run_id") + _positive_integer(row["session_start"], f"line {line_no}.session_start") + if isinstance(row["repeat"], bool) or not isinstance(row["repeat"], int) or row["repeat"] < 0: + raise ValueError(f"line {line_no}.repeat must be a non-negative integer") + + _nonempty_string(row["case_id"], f"line {line_no}.case_id") + if row["backend"] not in BACKENDS: + raise ValueError(f"line {line_no}.backend must be direct or diffusion") + if row["requested_provider"] not in PROVIDERS: + raise ValueError( + f"line {line_no}.requested_provider must be wasm or webgpu" + ) + observed = row["observed_provider"] + if observed is not None and observed not in PROVIDERS: + raise ValueError( + f"line {line_no}.observed_provider must be wasm, webgpu, or null" + ) + + _sha256(row["bundle_hash"], f"line {line_no}.bundle_hash") + _sha256(row["tokenizer_hash"], f"line {line_no}.tokenizer_hash") + _sha256(row["input_hash"], f"line {line_no}.input_hash") + noise_hash = _sha256( + row.get("noise_hash"), f"line {line_no}.noise_hash", allow_none=True + ) + if row["backend"] == "diffusion" and noise_hash is None: + raise ValueError( + f"line {line_no}.noise_hash is required for diffusion backend" + ) + + _positive_integer(row["candidate_count"], f"line {line_no}.candidate_count") + _positive_integer( + row["padded_option_slots"], f"line {line_no}.padded_option_slots" + ) + _positive_integer(row["sequence_length"], f"line {line_no}.sequence_length") + if "seed" not in row: + raise ValueError(f"line {line_no}: missing required field seed") + seed_value = row["seed"] + if seed_value is None: + if row["backend"] != "direct" or noise_hash is not None: + raise ValueError( + f"line {line_no}.seed may only be null for direct backend with " + "noise_hash=null (deterministic-direct; seed is not_applicable); " + "diffusion or direct-with-noise require a non-negative integer seed" + ) + else: + _nonnegative_integer(seed_value, f"line {line_no}.seed") + if row["candidate_count"] > row["padded_option_slots"]: + raise ValueError( + f"line {line_no}: candidate_count must be <= padded_option_slots" + ) + if row["padded_option_slots"] > MAX_PADDED_OPTION_SLOTS: + raise ValueError( + f"line {line_no}: padded_option_slots must be <= {MAX_PADDED_OPTION_SLOTS}" + ) + if row["sequence_length"] > MAX_SEQUENCE_LENGTH: + raise ValueError( + f"line {line_no}: sequence_length must be <= {MAX_SEQUENCE_LENGTH}" + ) + _nonempty_string(row["ort_version"], f"line {line_no}.ort_version") + + status = row["status"] + if status not in {"ok", "error", "timeout"}: + raise ValueError( + f"line {line_no}.status must be ok, error, or timeout" + ) + error = row.get("error") + if status in {"error", "timeout"}: + if error is None: + pass + elif not isinstance(error, str) or not error: + raise ValueError( + f"line {line_no}.error must be a non-empty string for status {status}" + ) + elif status == "ok" and error is not None: + raise ValueError( + f"line {line_no}.error must be null for successful samples" + ) + + for field in TIMING_FIELDS: + timing = _finite_number( + row.get(field), f"line {line_no}.{field}", allow_none=True + ) + if status == "ok" and timing is None: + raise ValueError( + f"line {line_no}: successful sample requires {field}" + ) + + for field in HEAP_FIELDS: + _finite_number(row.get(field), f"line {line_no}.{field}", allow_none=True) + + environment = provenance["environment"] + if not isinstance(environment, dict): + raise TypeError(f"line {line_no}.provenance.environment must be an object") + _nonempty_string( + provenance["execution_provider_evidence"], + f"line {line_no}.provenance.execution_provider_evidence", + ) + _positive_integer( + provenance["warmup_excluded_per_session"], + f"line {line_no}.provenance.warmup_excluded_per_session", + ) + _nonempty_string( + provenance["session_start_semantics"], + f"line {line_no}.provenance.session_start_semantics", + ) + + +def _iter_jsonl(path: Path) -> Iterator[tuple[int, dict[str, Any]]]: + with path.open("r", encoding="utf-8") as handle: + line_no = 0 + for raw in handle: + line_no += 1 + stripped = raw.strip() + if not stripped: + continue + yield line_no, json.loads(stripped) + + +def _condition_tuple(row: dict[str, Any]) -> tuple[Any, ...]: + provenance_sig = _provenance_condition_signature(row["provenance"]) + return ( + tuple(row[key] for key in CONDITION_KEYS), + provenance_sig["warmup_excluded_per_session"], + provenance_sig["session_start_semantics"], + json.dumps(provenance_sig["environment"], sort_keys=True), + provenance_sig["execution_provider_evidence"], + ) + + +def validate_and_aggregate(input_path: Path) -> dict[str, Any]: + if not input_path.exists(): + raise FileNotFoundError(f"input file not found: {input_path}") + + rows: list[dict[str, Any]] = [] + seen_identities: set[tuple[str, int, int]] = set() + first_condition: tuple[Any, ...] | None = None + provenance_reference: dict[str, Any] | None = None + session_samples: dict[int, int] = {} + success_count = 0 + failure_count = 0 + total_rows = 0 + + stage_timings: dict[str, list[float]] = {field: [] for field in TIMING_FIELDS} + heap_values: dict[str, list[float]] = {field: [] for field in HEAP_FIELDS} + + for line_no, row in _iter_jsonl(input_path): + _validate_row(row, line_no) + total_rows += 1 + identity = (row["run_id"], row["session_start"], row["repeat"]) + if identity in seen_identities: + raise ValueError( + f"duplicate (run_id, session_start, repeat) identity at line {line_no}: {identity}" + ) + seen_identities.add(identity) + + condition = _condition_tuple(row) + if first_condition is None: + first_condition = condition + provenance_reference = _provenance_condition_signature( + row["provenance"] + ) + elif condition != first_condition: + raise ValueError( + f"line {line_no}: mixed experimental conditions detected; " + "bundle, tokenizer, shape, provider, runtime, environment, " + "and config provenance fields must be homogeneous across all rows" + ) + + session_start = row["session_start"] + session_samples[session_start] = session_samples.get(session_start, 0) + 1 + + status = row["status"] + if status == "ok": + success_count += 1 + for field in TIMING_FIELDS: + stage_timings[field].append(float(row[field])) + for field in HEAP_FIELDS: + value = row.get(field) + if value is not None: + heap_values[field].append(float(value)) + else: + failure_count += 1 + + rows.append(row) + + if total_rows == 0: + raise ValueError("input JSONL contains no samples") + + return _build_summary( + rows=rows, + input_path=input_path, + provenance_reference=provenance_reference, + first_condition=first_condition, + session_samples=session_samples, + success_count=success_count, + failure_count=failure_count, + total_rows=total_rows, + stage_timings=stage_timings, + heap_values=heap_values, + ) + + +def _build_summary( + *, + rows: list[dict[str, Any]], + input_path: Path, + provenance_reference: dict[str, Any] | None, + first_condition: tuple[Any, ...] | None, + session_samples: dict[int, int], + success_count: int, + failure_count: int, + total_rows: int, + stage_timings: dict[str, list[float]], + heap_values: dict[str, list[float]], +) -> dict[str, Any]: + assert provenance_reference is not None + assert first_condition is not None + assert rows + + first_row = rows[0] + unique_sessions = len(session_samples) + warmup_per_session = provenance_reference["warmup_excluded_per_session"] + timed_requests_total = success_count + failure_count + complete = ( + unique_sessions >= 5 + and warmup_per_session >= 10 + and timed_requests_total >= 100 + ) + incomplete_reasons: list[str] = [] + if unique_sessions < 5: + incomplete_reasons.append( + f"only {unique_sessions} independent session starts observed (>= 5 required)" + ) + if warmup_per_session < 10: + incomplete_reasons.append( + f"warmup_excluded_per_session={warmup_per_session} (>= 10 required)" + ) + if timed_requests_total < 100: + incomplete_reasons.append( + f"only {timed_requests_total} timed requests total (>= 100 required)" + ) + + process_cold_claimable = False + + requested_provider = first_row["requested_provider"] + observed_provider = first_row["observed_provider"] + requested_is_proven = False + + summary_timings: dict[str, dict[str, float | None]] = {} + for field in TIMING_FIELDS: + values = stage_timings[field] + summary_timings[field] = { + "p50": percentile(values, 0.5), + "p95": percentile(values, 0.95), + } + + heap_maxima: dict[str, float | None] = {} + for field in HEAP_FIELDS: + values = heap_values[field] + heap_maxima[field] = max(values) if values else None + + per_session_samples: dict[str, int] = { + str(session_start): count + for session_start, count in sorted(session_samples.items()) + } + + summary: dict[str, Any] = { + "schema": "vons.browser-benchmark-summary/v1", + "generated_from": str(input_path), + "input_sha256": _file_sha256(input_path), + "experimental_condition": { + "run_id": first_row["run_id"], + "backend": first_row["backend"], + "requested_provider": requested_provider, + "observed_provider": observed_provider, + "bundle_hash": first_row["bundle_hash"], + "tokenizer_hash": first_row["tokenizer_hash"], + "candidate_count": first_row["candidate_count"], + "padded_option_slots": first_row["padded_option_slots"], + "sequence_length": first_row["sequence_length"], + "seed": first_row["seed"], + "ort_version": first_row["ort_version"], + "provenance": provenance_reference, + }, + "provider_claim_note": ( + "requested_provider is not represented as proven in this v1 summary. " + "observed_provider preserves the browser-reported value and " + "execution_provider_evidence preserves the raw evidence string, " + "but no structured verified execution artifact exists to elevate " + "a match to a proven claim; requested_is_proven is always False in v1." + ), + "process_cold_note": ( + "process-level cold start identity is not verifiable from page-only " + "browser observations in this v1 schema; session_start_semantics is " + "preserved as a raw label but process_cold_claimable is always False " + "regardless of semantics string content." + ), + "sample_counts": { + "total_rows": total_rows, + "successful": success_count, + "failed": failure_count, + "unique_session_starts": unique_sessions, + "per_session_samples": per_session_samples, + }, + "stage_percentiles_ms": summary_timings, + "heap_maxima_bytes": heap_maxima, + "measurement_completeness": { + "complete": complete, + "required_min_session_starts": 5, + "required_min_warmup_excluded_per_session": 10, + "required_min_timed_requests_total": 100, + "actual_session_starts": unique_sessions, + "actual_warmup_excluded_per_session": warmup_per_session, + "actual_timed_requests_total": timed_requests_total, + "incomplete_reasons": incomplete_reasons, + }, + "process_cold_claimable": bool(process_cold_claimable), + "requested_is_proven": bool(requested_is_proven), + } + return summary + + +def _iter_report_samples( + report: dict[str, Any], report_path: Path +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Return (top_provenance, normalized_rows) from a v1 browser-benchmark report. + + Normalizes current report field names to the JSONL schema used by the + validator: live_candidates -> candidate_count, allocated_candidates -> + padded_option_slots, sequence_length preferred with sequence_tokens as + fallback alias. Never invents missing values. Raises ValueError on any + required field that cannot be resolved. + """ + if not isinstance(report, dict): + raise TypeError("browser benchmark report must be a JSON object") + if report.get("schema") != "vons.browser-benchmark/v1": + raise ValueError("unexpected browser benchmark schema") + + _nonempty_string(report.get("run_id"), f"{report_path.name}.run_id") + backend = report.get("backend") + if backend not in BACKENDS: + raise ValueError(f"{report_path.name}.backend must be direct or diffusion") + requested_provider = report.get("requested_provider") + if requested_provider not in PROVIDERS: + raise ValueError( + f"{report_path.name}.requested_provider must be wasm or webgpu" + ) + environment = report.get("environment") + if not isinstance(environment, dict): + raise TypeError( + f"{report_path.name}.environment is required as provenance input" + ) + execution_provider_evidence = _nonempty_string( + report.get("execution_provider_evidence"), + f"{report_path.name}.execution_provider_evidence", + ) + warmup_excluded_per_session = _positive_integer( + report.get("warmup_excluded_per_session"), + f"{report_path.name}.warmup_excluded_per_session", + ) + session_start_semantics_raw = report.get("session_start_semantics") + if session_start_semantics_raw is None: + raise ValueError( + f"{report_path.name}.session_start_semantics is required to build " + "per-row provenance; do not invent missing values." + ) + session_start_semantics = _nonempty_string( + session_start_semantics_raw, + f"{report_path.name}.session_start_semantics", + ) + top_provenance = { + "environment": environment, + "execution_provider_evidence": execution_provider_evidence, + "warmup_excluded_per_session": warmup_excluded_per_session, + "session_start_semantics": session_start_semantics, + } + + samples = report.get("samples") + if not isinstance(samples, list): + raise TypeError(f"{report_path.name}.samples must be a list") + if not samples: + raise ValueError(f"{report_path.name}.samples contains no rows") + + ort_version = report.get("ort_version") + ort_version_is_str = isinstance(ort_version, str) and bool(ort_version) + if not ort_version_is_str: + runtime_info = report.get("runtime_info") + if isinstance(runtime_info, dict): + rt_ort = runtime_info.get("ortVersion") + if isinstance(rt_ort, str) and rt_ort: + ort_version = rt_ort + ort_version_is_str = True + + normalized: list[dict[str, Any]] = [] + for idx, sample in enumerate(samples, start=1): + if not isinstance(sample, dict): + raise TypeError(f"{report_path.name} sample {idx} must be an object") + + live_candidates = sample.get("live_candidates") + if live_candidates is None: + raise ValueError( + f"{report_path.name} sample {idx} is missing live_candidates " + "(source for candidate_count normalization)" + ) + allocated_candidates = sample.get("allocated_candidates") + if allocated_candidates is None: + raise ValueError( + f"{report_path.name} sample {idx} is missing allocated_candidates " + "(source for padded_option_slots normalization)" + ) + sequence_length = sample.get("sequence_length") + if sequence_length is None: + sequence_length = sample.get("sequence_tokens") + if sequence_length is None: + raise ValueError( + f"{report_path.name} sample {idx} is missing sequence_length " + "(sequence_tokens not present as alias fallback)" + ) + sample_ort_version = sample.get("ort_version") + if not (isinstance(sample_ort_version, str) and sample_ort_version): + if ort_version_is_str: + sample_ort_version = ort_version + else: + raise ValueError( + f"{report_path.name} sample {idx} is missing ort_version and " + "report does not provide a top-level ort_version fallback " + "(checked both ort_version and runtime_info.ortVersion)" + ) + sample_backend = sample.get("backend") + noise_hash = sample.get("noise_hash") + sample_seed = sample.get("seed") + if sample_seed is None and sample_backend == "direct" and noise_hash is None: + resolved_seed = None + else: + if "seed" not in sample: + raise ValueError( + f"{report_path.name} sample {idx} is missing seed; " + "null seed is only permitted for direct backend with " + "noise_hash=null (deterministic-direct, not_applicable)" + ) + resolved_seed = sample_seed + error_value = sample.get("error") + observed_provider = sample.get("observed_provider") + + mapped: dict[str, Any] = { + "run_id": sample["run_id"], + "session_start": sample["session_start"], + "repeat": sample["repeat"], + "case_id": sample["case_id"], + "backend": sample_backend, + "requested_provider": sample["requested_provider"], + "observed_provider": observed_provider, + "bundle_hash": sample["bundle_hash"], + "tokenizer_hash": sample["tokenizer_hash"], + "input_hash": sample["input_hash"], + "noise_hash": noise_hash, + "candidate_count": live_candidates, + "padded_option_slots": allocated_candidates, + "sequence_length": sequence_length, + "seed": resolved_seed, + "ort_version": sample_ort_version, + "status": sample["status"], + "error": error_value, + "provenance": top_provenance, + } + for field in TIMING_FIELDS: + mapped[field] = sample.get(field) + for field in HEAP_FIELDS: + if field in sample: + mapped[field] = sample.get(field) + normalized.append(mapped) + return top_provenance, normalized + + +def validate_and_aggregate_report(report_path: Path) -> dict[str, Any]: + """Entry point for --report-input: loads an existing v1 browser-benchmark + JSON report, normalizes its samples to the JSONL schema, then runs the + same validation and aggregation used by validate_and_aggregate. + """ + if not report_path.exists(): + raise FileNotFoundError(f"report file not found: {report_path}") + report = json.loads(report_path.read_text(encoding="utf-8")) + provenance_reference, normalized_rows = _iter_report_samples(report, report_path) + + total_rows = 0 + seen_identities: set[tuple[str, int, int]] = set() + first_condition: tuple[Any, ...] | None = None + session_samples: dict[int, int] = {} + success_count = 0 + failure_count = 0 + stage_timings: dict[str, list[float]] = {field: [] for field in TIMING_FIELDS} + heap_values: dict[str, list[float]] = {field: [] for field in HEAP_FIELDS} + accepted_rows: list[dict[str, Any]] = [] + + for line_no, row in enumerate(normalized_rows, start=1): + _validate_row(row, line_no) + total_rows += 1 + identity = (row["run_id"], row["session_start"], row["repeat"]) + if identity in seen_identities: + raise ValueError( + f"duplicate (run_id, session_start, repeat) identity in sample {line_no}: {identity}" + ) + seen_identities.add(identity) + condition = _condition_tuple(row) + if first_condition is None: + first_condition = condition + elif condition != first_condition: + raise ValueError( + f"sample {line_no}: mixed experimental conditions detected in report" + ) + session_start = row["session_start"] + session_samples[session_start] = session_samples.get(session_start, 0) + 1 + if row["status"] == "ok": + success_count += 1 + for field in TIMING_FIELDS: + stage_timings[field].append(float(row[field])) + for field in HEAP_FIELDS: + value = row.get(field) + if value is not None: + heap_values[field].append(float(value)) + else: + failure_count += 1 + accepted_rows.append(row) + + if total_rows == 0: + raise ValueError("report samples yielded no rows after normalization") + + return _build_summary( + rows=accepted_rows, + input_path=report_path, + provenance_reference=provenance_reference, + first_condition=first_condition, + session_samples=session_samples, + success_count=success_count, + failure_count=failure_count, + total_rows=total_rows, + stage_timings=stage_timings, + heap_values=heap_values, + ) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--input", type=Path, + help="JSONL file of raw browser benchmark samples") + group.add_argument("--report-input", type=Path, + help="Existing vons.browser-benchmark/v1 JSON report; " + "normalizes live_candidates/allocated_candidates/" + "sequence_tokens to JSONL schema and carries top-level provenance") + parser.add_argument("--output", type=Path, required=True, + help="destination path for the summary JSON") + args = parser.parse_args() + + if args.input is not None: + summary = validate_and_aggregate(args.input) + else: + summary = validate_and_aggregate_report(args.report_input) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +if __name__ == "__main__": + main() diff --git a/tools/summarize_quality.py b/tools/summarize_quality.py new file mode 100644 index 0000000000000000000000000000000000000000..4e8ca831c5a593f60e183dcf6b8d621352ca0501 --- /dev/null +++ b/tools/summarize_quality.py @@ -0,0 +1,170 @@ +"""Recompute quality metrics from saved Vons prediction reports. + +This tool never fabricates unavailable external-task results. Its current +reports are explicitly labeled synthetic-pilot measurements. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import statistics +from collections import defaultdict +from pathlib import Path +from typing import Any + +from vons.data import read_jsonl +from vons.evaluation import Prediction, summarize + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_report(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict) or not isinstance(value.get("predictions"), list): + raise TypeError(f"{path} must contain a predictions list") + return value + + +def to_predictions(raw: list[dict[str, Any]], path: Path) -> list[Prediction]: + predictions: list[Prediction] = [] + seen: set[str] = set() + for index, value in enumerate(raw, start=1): + if not isinstance(value, dict): + raise TypeError(f"{path}: prediction {index} must be an object") + example_id = str(value.get("example_id", "")) + if not example_id or example_id in seen: + raise ValueError(f"{path}: duplicate or missing example_id at row {index}") + seen.add(example_id) + probabilities = value.get("probabilities") + if not isinstance(probabilities, dict): + raise TypeError(f"{path}: probabilities must be an object at row {index}") + numeric_probabilities = {str(key): float(item) for key, item in probabilities.items()} + confidence = float(value.get("confidence")) + if not math.isfinite(confidence) or any(not math.isfinite(item) for item in numeric_probabilities.values()): + raise ValueError(f"{path}: non-finite probability or confidence at row {index}") + predictions.append( + Prediction( + example_id=example_id, + choice=None if value.get("choice") is None else str(value["choice"]), + probabilities=numeric_probabilities, + confidence=confidence, + abstained=bool(value.get("abstained", False)), + latency_ms=None if value.get("latency_ms") is None else float(value["latency_ms"]), + error=None if value.get("error") is None else str(value["error"]), + ) + ) + return predictions + + +def summarize_report(data_path: Path, report_path: Path) -> dict[str, Any]: + examples = read_jsonl(data_path) + report = load_report(report_path) + predictions = to_predictions(report["predictions"], report_path) + expected_ids = {row.id for row in examples} + prediction_ids = {item.example_id for item in predictions} + summary = summarize(examples, predictions) + config = report.get("config", {}) + if not isinstance(config, dict): + raise TypeError(f"{report_path}: config must be an object") + return { + "report": str(report_path), + "report_sha256": file_sha256(report_path), + "backend": config.get("backend"), + "seed": config.get("seed"), + "model_id": config.get("model_id"), + "dataset": str(data_path), + "dataset_sha256": file_sha256(data_path), + "dataset_rows": len(examples), + "prediction_rows": len(predictions), + "missing_prediction_rows": len(expected_ids - prediction_ids), + "extra_prediction_rows": len(prediction_ids - expected_ids), + "answerable_rows": sum(row.answerable for row in examples), + "answerable_prediction_rows": sum(row.answerable and row.id in prediction_ids for row in examples), + "abstained_rows": sum(item.abstained for item in predictions), + "error_rows": sum(item.error is not None for item in predictions), + "metrics": summary, + "measurement_label": "measured_synthetic_pilot", + "quality_scope": "synthetic test split only; not Mind2Web or browser task success", + } + + +def descriptive_seed_stats(rows: list[dict[str, Any]]) -> dict[str, Any]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + grouped[str(row["backend"])].append(row) + fields = ("accuracy_answerable", "macro_f1_answerable", "coverage", "selective_risk", "nll_answerable", "brier_answerable", "ece") + result: dict[str, Any] = {} + for backend, backend_rows in sorted(grouped.items()): + result[backend] = {} + for field in fields: + values = [row["metrics"].get(field) for row in backend_rows] + defined = [float(value) for value in values if value is not None] + result[backend][field] = { + "mean": statistics.mean(defined) if defined else None, + "min": min(defined) if defined else None, + "max": max(defined) if defined else None, + "seed_count": len(backend_rows), + "defined_seed_count": len(defined), + "null_seed_count": len(values) - len(defined), + "interpretation": "descriptive across saved seeds; not an independent test population", + } + return result + + +def build_quality_summary(data_path: Path, report_paths: list[Path], *, version: str = "v1") -> dict[str, Any]: + if not report_paths: + raise ValueError("at least one report is required") + rows = [summarize_report(data_path, path) for path in report_paths] + if version not in {"v1", "v2"}: + raise ValueError("version must be v1 or v2") + result = { + "schema": f"vons.quality-summary/{version}", + "measurement_label": "measured_synthetic_pilot", + "dataset": {"path": str(data_path), "sha256": file_sha256(data_path), "rows": len(read_jsonl(data_path))}, + "reports": rows, + "descriptive_seed_stats": descriptive_seed_stats(rows), + "external_evaluation": { + "mind2web": { + "status": "not_evaluated", + "candidate_recall": None, + "conditional_selection_accuracy": None, + "reason": "No Mind2Web evaluation report was supplied to this synthetic summary.", + }, + "jev": {"status": "not_run", "agreement": None}, + }, + "disclosure": "These numbers are recomputed from saved synthetic-pilot predictions. They do not establish external-task quality, browser task success, or generalization.", + } + if version == "v2": + result["metric_version"] = "vons.metrics/v2" + result["input_sha256"] = { + "dataset": file_sha256(data_path), + "reports": {str(path): file_sha256(path) for path in report_paths}, + } + result["disclosure"] += " Version 2 also records every input digest and uses the vons.metrics/v2 definitions present in each saved report." + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data", type=Path, required=True) + parser.add_argument("--report", type=Path, action="append", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--version", choices=("v1", "v2"), default="v1") + args = parser.parse_args() + result = build_quality_summary(args.data, args.report, version=args.version) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tools/verify_browser_benchmark.py b/tools/verify_browser_benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..9d9af599ac608bd46b15503933af08554a1156a1 --- /dev/null +++ b/tools/verify_browser_benchmark.py @@ -0,0 +1,297 @@ +"""Validate and recompute a saved Vons browser benchmark report. + +The browser page is responsible for collecting raw samples. This tool is the +second, offline boundary: it rejects malformed samples and recomputes summary +statistics instead of trusting the values emitted by the page. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +from pathlib import Path +from typing import Any + +TIMING_FIELDS = ( + "tokenizeMs", + "prepareFeedMs", + "inferenceAndReadbackMs", + "postprocessMs", + "totalRequestMs", +) +PROVIDERS = {"wasm", "webgpu"} +BACKENDS = {"direct", "diffusion"} +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _finite_number(value: Any, label: str, *, allow_none: bool = False) -> float | None: + if value is None and allow_none: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{label} must be a finite number or null") + numeric = float(value) + if not math.isfinite(numeric) or numeric < 0: + raise ValueError(f"{label} must be a finite non-negative number") + return numeric + + +def _positive_integer(value: Any, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{label} must be a positive integer") + return value + + +def _nonnegative_integer(value: Any, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{label} must be a non-negative integer") + return value + + +def _sha256(value: Any, label: str, *, allow_none: bool = False) -> str | None: + if value is None and allow_none: + return None + if not isinstance(value, str) or SHA256_PATTERN.fullmatch(value) is None: + raise ValueError(f"{label} must be a lowercase SHA-256 hex string" ) + return value + + +def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + index = (len(ordered) - 1) * fraction + lower = math.floor(index) + upper = math.ceil(index) + if lower == upper: + return ordered[lower] + return ordered[lower] + (ordered[upper] - ordered[lower]) * (index - lower) + + +def _assert_close(actual: Any, expected: float | None, label: str) -> None: + if expected is None: + if actual is not None: + raise ValueError(f"{label} must be null, got {actual!r}") + return + if not isinstance(actual, (int, float)) or not math.isclose( + float(actual), expected, rel_tol=1e-9, abs_tol=1e-9 + ): + raise ValueError(f"{label} does not match recomputed value {expected!r}: {actual!r}") + + +def verify_report(report_path: Path, *, allow_incomplete: bool = False) -> dict[str, Any]: + report = json.loads(report_path.read_text(encoding="utf-8")) + if not isinstance(report, dict): + raise TypeError("browser benchmark report must be an object") + if report.get("schema") != "vons.browser-benchmark/v1": + raise ValueError("unexpected browser benchmark schema") + + sessions = _positive_integer(report.get("sessions"), "sessions") + warmup = _positive_integer(report.get("warmup_excluded_per_session"), "warmup_excluded_per_session") + repeats = _positive_integer(report.get("repeats_per_session"), "repeats_per_session") + _positive_integer(report.get("cases"), "cases") + if not isinstance(report.get("run_id"), str) or not report["run_id"]: + raise TypeError("run_id is required") + backend = report.get("backend") + if backend not in BACKENDS: + raise ValueError("backend must be direct or diffusion") + provider = report.get("requested_provider") + if provider not in PROVIDERS: + raise ValueError("requested_provider must be wasm or webgpu") + environment = report.get("environment") + if ( + not isinstance(environment, dict) + or not isinstance(environment.get("user_agent"), str) + or not environment["user_agent"] + ): + raise TypeError("environment.user_agent is required") + if not isinstance(environment.get("cross_origin_isolated"), bool): + raise TypeError("environment.cross_origin_isolated must be boolean") + runtime_info = report.get("runtime_info") + if runtime_info is not None and not isinstance(runtime_info, dict): + raise TypeError("runtime_info must be an object or null") + if not isinstance(report.get("execution_provider_evidence"), str) or not report["execution_provider_evidence"]: + raise TypeError("execution_provider_evidence is required") + samples = report.get("samples") + session_records = report.get("session_records") + if not isinstance(samples, list) or not isinstance(session_records, list): + raise TypeError("samples and session_records must be lists") + + seen_slots: set[tuple[int, int]] = set() + successful: list[dict[str, Any]] = [] + failures = 0 + observed_providers: set[str] = set() + for index, sample in enumerate(samples, start=1): + if not isinstance(sample, dict): + raise TypeError(f"sample {index} must be an object") + session_start = sample.get("session_start") + repeat = sample.get("repeat") + if ( + isinstance(session_start, bool) + or not isinstance(session_start, int) + or not 1 <= session_start <= sessions + ): + raise ValueError(f"sample {index} has an invalid session_start") + if isinstance(repeat, bool) or not isinstance(repeat, int) or not 0 <= repeat < repeats: + raise ValueError(f"sample {index} has an invalid repeat") + slot = (session_start, repeat) + if slot in seen_slots: + raise ValueError(f"duplicate sample slot {slot}") + seen_slots.add(slot) + if sample.get("requested_provider") != provider: + raise ValueError(f"sample {index} requested_provider disagrees with report") + if sample.get("backend") != backend: + raise ValueError(f"sample {index} backend disagrees with report") + if not isinstance(sample.get("run_id"), str) or not sample["run_id"]: + raise TypeError(f"sample {index}.run_id is required") + if not isinstance(sample.get("case_id"), str) or not sample["case_id"]: + raise TypeError(f"sample {index}.case_id is required") + _sha256(sample.get("bundle_hash"), f"sample {index}.bundle_hash") + _sha256(sample.get("tokenizer_hash"), f"sample {index}.tokenizer_hash") + _sha256(sample.get("input_hash"), f"sample {index}.input_hash") + noise_hash = _sha256(sample.get("noise_hash"), f"sample {index}.noise_hash", allow_none=True) + if backend == "diffusion" and noise_hash is None: + raise ValueError(f"sample {index}.noise_hash is required for diffusion") + if backend == "direct" and noise_hash is not None: + raise ValueError(f"sample {index}.noise_hash must be null for direct") + observed = sample.get("observed_provider") + if observed is not None: + if observed not in PROVIDERS: + raise ValueError(f"sample {index} has an invalid observed_provider") + observed_providers.add(observed) + status = sample.get("status") + if status not in {"ok", "error"}: + raise ValueError(f"sample {index} status must be ok or error") + if not isinstance(sample.get("observed_provider_status"), str) or not sample["observed_provider_status"]: + raise ValueError(f"sample {index} requires observed_provider_status") + for field in TIMING_FIELDS: + timing = _finite_number(sample.get(field), f"sample {index}.{field}", allow_none=True) + if status == "ok" and timing is None: + raise ValueError(f"successful sample {index}.{field} must be numeric") + if status == "error" and timing is not None: + raise ValueError(f"failed sample {index}.{field} must be null") + shape_fields = ("live_candidates", "allocated_candidates", "live_tokens", "sequence_length") + for field in shape_fields: + value = sample.get(field) + if status == "ok": + _nonnegative_integer(value, f"sample {index}.{field}") + elif value is not None: + raise ValueError(f"failed sample {index}.{field} must be null") + if status == "ok" and sample["allocated_candidates"] < sample["live_candidates"]: + raise ValueError(f"sample {index} allocated_candidates is below live_candidates") + for field in ("js_heap_used_bytes_before", "js_heap_used_bytes_after"): + _finite_number(sample.get(field), f"sample {index}.{field}", allow_none=True) + if status == "ok": + successful.append(sample) + else: + failures += 1 + + session_record_ids: set[int] = set() + session_failures: list[int] = [] + for index, record in enumerate(session_records, start=1): + if not isinstance(record, dict): + raise TypeError(f"session record {index} must be an object") + session_start = record.get("session_start") + if ( + isinstance(session_start, bool) + or not isinstance(session_start, int) + or not 1 <= session_start <= sessions + ): + raise ValueError(f"session record {index} has an invalid session_start") + if session_start in session_record_ids: + raise ValueError(f"duplicate session record {session_start}") + session_record_ids.add(session_start) + _finite_number(record.get("load_ms"), f"session record {index}.load_ms") + if record.get("status") not in {"ok", "error"}: + raise ValueError(f"session record {index} status must be ok or error") + if record["status"] == "error": + session_failures.append(session_start) + + incomplete_reasons: list[str] = [] + if len(session_records) != sessions: + incomplete_reasons.append(f"expected {sessions} session records, got {len(session_records)}") + if session_failures: + incomplete_reasons.append(f"session creation failed for starts {session_failures}") + expected_samples = sessions * repeats + if len(samples) != expected_samples: + incomplete_reasons.append(f"expected {expected_samples} timed samples, got {len(samples)}") + if incomplete_reasons and not allow_incomplete: + raise ValueError("incomplete browser benchmark: " + "; ".join(incomplete_reasons)) + + total_values = [float(sample["totalRequestMs"]) for sample in successful] + inference_values = [float(sample["inferenceAndReadbackMs"]) for sample in successful] + before_values = [ + float(sample["js_heap_used_bytes_before"]) + for sample in successful + if sample.get("js_heap_used_bytes_before") is not None + ] + after_values = [ + float(sample["js_heap_used_bytes_after"]) + for sample in successful + if sample.get("js_heap_used_bytes_after") is not None + ] + recomputed_summary = { + "successful_samples": len(successful), + "failed_samples": failures, + "total_request_ms_p50": percentile(total_values, 0.5), + "total_request_ms_p95": percentile(total_values, 0.95), + "inference_and_readback_ms_p50": percentile(inference_values, 0.5), + "inference_and_readback_ms_p95": percentile(inference_values, 0.95), + "js_heap_before_max": max(before_values) if before_values else None, + "js_heap_after_max": max(after_values) if after_values else None, + } + supplied_summary = report.get("summary") + if supplied_summary is not None: + if not isinstance(supplied_summary, dict): + raise TypeError("summary must be an object") + for key, expected in recomputed_summary.items(): + _assert_close(supplied_summary.get(key), expected, f"summary.{key}") + + return { + "schema": "vons.browser-benchmark-verification/v1", + "measurement_label": "verified_browser_benchmark_raw_samples", + "complete": not incomplete_reasons, + "incomplete_reasons": incomplete_reasons, + "report": str(report_path), + "report_sha256": file_sha256(report_path), + "requested_provider": provider, + "observed_providers": sorted(observed_providers), + "sessions": sessions, + "warmup_excluded_per_session": warmup, + "repeats_per_session": repeats, + "sample_rows": len(samples), + "successful_samples": len(successful), + "failed_samples": failures, + "recomputed_summary": recomputed_summary, + "memory_note": "JS heap values are before/after observations only; null means unavailable; no GPU peak is inferred.", + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--allow-incomplete", + action="store_true", + help="validate and summarize a partial diagnostic run without treating it as complete", + ) + args = parser.parse_args() + result = verify_report(args.report, allow_incomplete=args.allow_incomplete) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tools/verify_browser_parity.py b/tools/verify_browser_parity.py new file mode 100644 index 0000000000000000000000000000000000000000..bffcea1659826e826a12cd6a0f37e2b8af5257eb --- /dev/null +++ b/tools/verify_browser_parity.py @@ -0,0 +1,523 @@ +"""Independently check browser traces against the pinned Python CPU ONNX graph. + +This consumes historical full-graph bundles, not the shared deployment format. +No output is overwritten. Score-head rejection is an unsupported capability, +never a successful numerical comparison. The current browser path does not +apply calibration; matching it must not be described as calibrated parity. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +from collections import Counter +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any + +import numpy as np + +from vons.bundle import verify_bundle_manifest + +HASH_PATTERN = re.compile(r"[0-9a-f]{64}") +SCHEMA = "vons.browser-parity/v1" + + +class ParityError(ValueError): + """A mismatch or missing evidence prevents a successful comparison.""" + + +class InputOverflow(ParityError): + """The independently tokenized question exceeds the declared budget.""" + + +def file_hash(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def stable_json(value: Any) -> str: + """Match the harness's JSON.stringify/localeCompare for its fixture domain. + + Python's Unicode sorting is not JS localeCompare. Restrict object keys to + lowercase ASCII letters (the complete current fixture domain), and numeric + values to safe integers. Unsupported collation/number formatting fails + closed instead of silently attesting to a differently canonicalized hash. + Unicode *values* are preserved, including the current multilingual states. + """ + if isinstance(value, dict): + if any(not isinstance(key, str) or not re.fullmatch(r"[a-z]+", key) for key in value): + raise ParityError("canonical request keys require lowercase ASCII letters") + return "{" + ",".join( + json.dumps(key) + ":" + stable_json(value[key]) for key in sorted(value) + ) + "}" + if isinstance(value, list): + return "[" + ",".join(stable_json(item) for item in value) + "]" + if isinstance(value, bool) or value is None or isinstance(value, str): + return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + if isinstance(value, (int, float)): + if not math.isfinite(value) or int(value) != value or abs(value) > 2**53 - 1: + raise ParityError("canonical request numbers must be finite safe integers") + return str(int(value)) + raise ParityError("request is not JSON-compatible") + + +def request_hash(request: dict[str, Any]) -> str: + return hashlib.sha256(stable_json(request).encode("utf-8")).hexdigest() + + +def _required_hash(value: Any, label: str) -> str: + if not isinstance(value, str) or HASH_PATTERN.fullmatch(value) is None: + raise ParityError(f"{label} requires a lowercase SHA-256 digest") + return value + + +def _object_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ParityError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def read_json(path: Path) -> Any: + def reject_constant(value: str) -> None: + raise ParityError(f"non-finite JSON constant: {value}") + + return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=_object_pairs, + parse_constant=reject_constant) + + +def _relative_file(root: Path, relative: Any) -> Path: + if not isinstance(relative, str) or not relative or "\\" in relative or ":" in relative: + raise ParityError("manifest file path must be a relative POSIX path") + value = PurePosixPath(relative) + if value.is_absolute() or PureWindowsPath(relative).drive or ".." in value.parts: + raise ParityError(f"manifest path escapes the bundle: {relative}") + current = root + for part in value.parts: + current = current / part + if current.is_symlink(): + raise ParityError(f"manifest path contains a symlink: {relative}") + if not current.is_file() or not current.resolve().is_relative_to(root.resolve()): + raise ParityError(f"manifest file is missing or outside bundle: {relative}") + return current + + +def validate_artifacts(report: dict[str, Any], manifest_path: Path) -> dict[str, Any]: + if not isinstance(report, dict): + raise ParityError("browser report must be an object") + if report.get("schema") != SCHEMA: + raise ParityError("unsupported browser report schema") + if report.get("requested_provider") not in {"wasm", "webgpu"}: + raise ParityError("requested_provider must identify wasm or webgpu") + expected = _required_hash(report.get("manifest_hash"), "manifest_hash") + if file_hash(manifest_path) != expected: + raise ParityError("browser manifest_hash does not match the supplied manifest bytes") + manifest = read_json(manifest_path) + if manifest.get("schema_version") != "vons.bundle.manifest/v1": + raise ParityError("only historical full-graph bundle manifests are supported") + backend = report.get("backend") + if backend not in {"direct", "diffusion"} or manifest.get("metadata", {}).get("backend") != backend: + raise ParityError("report backend does not match manifest") + files = manifest.get("files") + if not isinstance(files, list) or not files: + raise ParityError("manifest file inventory is empty") + for item in files: + if not isinstance(item, dict): + raise ParityError("manifest file entries must be objects") + path = _relative_file(manifest_path.parent, item.get("path")) + if type(item.get("bytes")) is not int or item["bytes"] < 0: + raise ParityError("manifest bytes must be a nonnegative integer") + if path.stat().st_size != item["bytes"] or file_hash(path) != _required_hash( + item.get("sha256"), f"file {item['path']}" + ): + raise ParityError(f"manifest file hash/bytes mismatch: {item['path']}") + # Also inspect actual ONNX external-data references and bundle byte totals. + verification = verify_bundle_manifest(manifest_path) + tokenizers = [item for item in files if item.get("role") == "tokenizer" + and item["path"].endswith("/tokenizer.json")] + tokenizers += [item for item in files if item.get("role") == "tokenizer" + and item["path"] == "tokenizer.json"] + graphs = [item for item in files if item.get("role") == "model_graph"] + if len(tokenizers) != 1 or len(graphs) != 1: + raise ParityError("manifest must select exactly one tokenizer.json and model graph") + if tokenizers[0]["sha256"] != _required_hash(report.get("tokenizer_hash"), "tokenizer_hash"): + raise ParityError("browser tokenizer_hash does not match the pinned tokenizer") + graph = _relative_file(manifest_path.parent, graphs[0]["path"]) + if graph.parent != manifest_path.parent: + raise ParityError("historical graph must be beside its manifest for external-data resolution") + return {"manifest": manifest, "verification": verification, "graph": graph, + "tokenizer": _relative_file(manifest_path.parent, tokenizers[0]["path"])} + + +def validate_request(request: Any) -> list[dict[str, Any]]: + if not isinstance(request, dict): + raise ParityError("request must be an object") + state = request.get("state") + if not isinstance(state, (str, dict)) or isinstance(state, str) and not state.strip(): + raise ParityError("state must be a nonempty string or an object") + questions = request.get("questions") + if not isinstance(questions, list) or not 1 <= len(questions) <= 8: + raise ParityError("questions must contain 1..8 items") + ids: set[str] = set() + for question in questions: + if not isinstance(question, dict): + raise ParityError("questions must contain objects") + key = question.get("id") + if not isinstance(key, str) or not key or key in ids: + raise ParityError("question IDs must be unique nonempty strings") + ids.add(key) + kind, options = question.get("type"), question.get("options") + if kind not in {"choice", "boolean", "score"}: + raise ParityError("unsupported question type") + if not isinstance(question.get("prompt"), str) or not question["prompt"].strip(): + raise ParityError("question prompt must be a nonempty string") + if not isinstance(options, list) or any(not isinstance(x, str) or not x.strip() + for x in options): + raise ParityError("options must contain nonempty strings") + if len(set(options)) != len(options): + raise ParityError("options must be unique") + if kind == "boolean": + if options not in ([], ["true", "false"]): + raise ParityError("boolean options must be true/false") + elif not 2 <= len(options) <= 32: + raise ParityError("candidate count must be 2..32") + rubric = question.get("rubric", []) + if not isinstance(rubric, list): + raise ParityError("rubric must be a list") + if kind == "score": + if not 2 <= len(rubric) <= 10 or any(not isinstance(x, str) for x in rubric): + raise ParityError("score requires 2..10 string rubric levels") + elif rubric: + raise ParityError("only score may provide rubric levels") + return questions + + +def candidate_options(question: dict[str, Any]) -> list[str]: + return ["true", "false"] if question["type"] == "boolean" and not question["options"] \ + else question["options"] + + +def build_inputs(tokenizer: Any, state: Any, question: dict[str, Any], + metadata: dict[str, Any], backend: str) -> tuple[dict[str, np.ndarray], dict[str, int]]: + options = candidate_options(question) + sequence = metadata.get("sequence_length", 512) + limit = metadata.get("option_count", 32) + if type(sequence) is not int or not 1 <= sequence <= 512: + raise ParityError("manifest sequence_length must be an integer in 1..512") + if type(limit) is not int or not len(options) <= limit <= 32: + raise ParityError("manifest option_count cannot accommodate the question") + slots = limit if backend == "diffusion" else len(options) + state_text = state if isinstance(state, str) else stable_json(state) + prefix = f"{state_text}\nQuestion: {question['prompt']}\n" + aggregate = tokenizer.encode(prefix + "Candidates:\n" + "\n".join(options)) + if len(aggregate.ids) > sequence: + raise InputOverflow(f"aggregate tokens {len(aggregate.ids)} exceed {sequence}") + encoded = [tokenizer.encode(prefix + "Candidate: " + option) for option in options] + if any(len(item.ids) > sequence for item in encoded): + raise InputOverflow(f"candidate tokens exceed {sequence}") + padding = tokenizer.token_to_id("[PAD]") + feed = {"input_ids": np.full((1, slots, sequence), padding or 0, dtype=np.int64), + "attention_mask": np.zeros((1, slots, sequence), dtype=np.int64), + "token_type_ids": np.zeros((1, slots, sequence), dtype=np.int64), + "option_mask": np.zeros((1, slots), dtype=np.bool_)} + for index, item in enumerate(encoded): + size = len(item.ids) + if len(item.attention_mask) != size or len(item.type_ids) != size: + raise ParityError("Python tokenizer returned inconsistent array lengths") + feed["input_ids"][0, index, :size] = item.ids + feed["attention_mask"][0, index, :size] = item.attention_mask + feed["token_type_ids"][0, index, :size] = item.type_ids + feed["option_mask"][0, index] = True + shape = {"live_candidates": len(options), "allocated_candidates": slots, + "live_tokens": sum(len(item.ids) for item in encoded), "sequence_length": sequence} + return feed, shape + + +def finite_array(value: Any, label: str) -> np.ndarray: + try: + array = np.asarray(value, dtype=np.float64) + except (ValueError, TypeError) as exc: + raise ParityError(f"{label} is not numeric") from exc + if not array.size or not np.isfinite(array).all(): + raise ParityError(f"{label} is empty or non-finite") + return array + + +def assert_close(actual: Any, expected: Any, label: str, atol: float, rtol: float) -> float: + left, right = finite_array(actual, label), finite_array(expected, label) + if left.shape != right.shape or not np.allclose(left, right, atol=atol, rtol=rtol, + equal_nan=False): + raise ParityError(f"{label} differs from Python reference (atol={atol}, rtol={rtol})") + return float(np.max(np.abs(left - right))) + + +def postprocess(scores: Any, answerability_logit: float, options: list[str], + question_id: str) -> dict[str, Any]: + values = finite_array(scores, "reference scores") + if values.shape != (len(options),): + raise ParityError("reference score count differs from candidates") + logit = float(finite_array([answerability_logit], "reference answerability")[0]) + exp = np.exp(values - np.max(values)) + probabilities = exp / exp.sum() + answerability = 1 / (1 + math.exp(-logit)) if logit >= 0 else math.exp(logit) / (1 + math.exp(logit)) + index = int(np.argmax(probabilities)) + confidence = float(probabilities[index] * answerability) + reason = "answerability_below_threshold" if answerability < 0.5 else ( + "confidence_below_threshold" if confidence < 0.55 else None + ) + return {"question_id": question_id, "choice": None if reason else options[index], + "probabilities": [{"option": option, "probability": float(probabilities[i])} + for i, option in enumerate(options)], + "confidence": confidence, "status": "abstain" if reason else "ok", + "abstain_reason": reason} + + +def compare_answer(actual: Any, expected: dict[str, Any], atol: float, rtol: float) -> None: + if not isinstance(actual, dict): + raise ParityError("answer is missing") + for field in ("question_id", "choice", "status", "abstain_reason"): + if field not in actual or actual[field] != expected[field]: + raise ParityError(f"answer {field} differs from Python reference") + assert_close([actual.get("confidence")], [expected["confidence"]], "confidence", atol, rtol) + probabilities = actual.get("probabilities") + if not isinstance(probabilities, list) or len(probabilities) != len(expected["probabilities"]): + raise ParityError("answer probability count differs") + for left, right in zip(probabilities, expected["probabilities"], strict=True): + if not isinstance(left, dict) or left.get("option") != right["option"]: + raise ParityError("answer probability options differ") + assert_close([item.get("probability") for item in probabilities], + [item["probability"] for item in expected["probabilities"]], + "answer probabilities", atol, rtol) + + +class CpuReference: + def __init__(self, graph: Path): + import onnxruntime as ort + + options = ort.SessionOptions() + options.intra_op_num_threads = 1 + options.inter_op_num_threads = 1 + self.session = ort.InferenceSession(str(graph), options, providers=["CPUExecutionProvider"]) + self.version = ort.__version__ + + def run(self, feed: dict[str, np.ndarray], backend: str) -> tuple[np.ndarray, np.ndarray]: + scores, answerability = self.session.run( + ["scores" if backend == "diffusion" else "logits", "answerability"], feed + ) + return scores, answerability + + +def verify_trace(trace: dict[str, Any], timing: dict[str, Any], response_answer: Any, + request: dict[str, Any], question: dict[str, Any], tokenizer: Any, + metadata: dict[str, Any], backend: str, noise_values: Any, + reference: Any, atol: float, rtol: float) -> dict[str, Any]: + if trace.get("questionId") != question["id"] or timing.get("questionId") != question["id"]: + raise ParityError("trace/timing question order or ID differs") + feed, shape = build_inputs(tokenizer, request["state"], question, metadata, backend) + for field, expected in shape.items(): + if type(timing.get(field)) is not int or timing[field] != expected: + raise ParityError(f"timing shape {field} differs from independent tokenization") + for browser_key, python_key in (("inputIds", "input_ids"), ("attentionMask", "attention_mask"), + ("tokenTypeIds", "token_type_ids"), ("optionMask", "option_mask")): + actual = trace.get(browser_key) + if not isinstance(actual, list) or any(type(item) is not int for item in actual): + raise ParityError(f"{browser_key} must be an integer array") + if actual != feed[python_key].astype(np.int64).ravel().tolist(): + raise ParityError(f"{browser_key} differs from independent tokenizer/padding") + noise_hash = None + if backend == "diffusion": + expected_shape = (shape["allocated_candidates"],) + noise = finite_array(trace.get("initialNoise"), "initialNoise") + frozen = finite_array(noise_values, "report noise_values") + if noise.shape != expected_shape or frozen.shape != expected_shape: + raise ParityError("initial noise dimensions differ from candidate slots") + # Both sources must represent the same exact stored Float32 values. + if not np.array_equal(noise, noise.astype(np.float32).astype(np.float64)): + raise ParityError("trace initialNoise is not an exact Float32 array") + if not np.array_equal(noise, frozen): + raise ParityError("trace noise differs from report's frozen noise_values") + noise32 = noise.astype(" dict[str, Any]: + if any(not math.isfinite(x) or x < 0 for x in (atol, rtol)): + raise ParityError("tolerances must be finite and nonnegative") + records = report.get("records") + if not isinstance(records, list) or not records or report.get("fixture_count") != len(records): + raise ParityError("report requires nonempty records and matching fixture_count") + backend, metadata = report["backend"], manifest["metadata"] + results: list[dict[str, Any]] = [] + seen: set[str] = set() + for row in records: + if not isinstance(row, dict): + raise ParityError("fixture records must be objects") + result: dict[str, Any] = {"fixture_id": row.get("fixture_id"), "questions": []} + results.append(result) + try: + key = row.get("fixture_id") + if not isinstance(key, str) or not key or key in seen: + raise ParityError("fixture IDs must be unique nonempty strings") + seen.add(key) + request = row.get("request") + fingerprint = _required_hash(row.get("input_hash"), "input_hash") + if request_hash(request) != fingerprint: + raise ParityError("input_hash differs from canonical request") + result["input_hash"] = fingerprint + questions = validate_request(request) + status = row.get("status") + traces, timings = row.get("traces"), row.get("timings") + if not isinstance(traces, list) or not isinstance(timings, list): + raise ParityError("trace and timing arrays are required, including for errors") + if status == "expected_error": + if traces or timings or not isinstance(row.get("error"), str) or not row["error"]: + raise ParityError("expected errors require an error and empty trace/timing arrays") + if any(question["type"] == "score" for question in questions): + if "score questions are not supported" not in row["error"]: + raise ParityError("score rejection does not match the recorded SDK limitation") + result.update(status="unverified", reason="score_head_unsupported", + numerical_comparison=False, contract_error_verified=False) + continue + overflow = None + for question in questions: + try: + build_inputs(tokenizer, request["state"], question, metadata, backend) + except InputOverflow as exc: + overflow = str(exc) + break + if overflow is None or not any(text in row["error"].lower() + for text in ("token", "budget", "overflow")): + raise ParityError("expected overflow could not be independently confirmed") + result.update(status="contract_error_verified", reason=overflow, + numerical_comparison=False, contract_error_verified=True) + continue + if status != "ok": + raise ParityError(f"browser request did not succeed: {row.get('error', status)}") + if any(question["type"] == "score" for question in questions): + raise ParityError("current full-graph reference does not implement score questions") + response = row.get("response") + if not isinstance(response, dict) or response.get("backend") != backend: + raise ParityError("response backend is missing or mismatched") + expected_model_id = metadata.get("model_id", f"vons-{backend}-onnx-web") + if response.get("model_id") != expected_model_id: + raise ParityError("response model_id differs from the supplied manifest") + answers = response.get("answers") + if not isinstance(answers, list) or not len(traces) == len(timings) == len(answers) == len(questions): + raise ParityError("every question requires exactly one trace, timing and answer") + for question, trace, timing, answer in zip(questions, traces, timings, answers, strict=True): + item = {"question_id": question["id"], "status": "failed"} + result["questions"].append(item) + try: + item.update(verify_trace(trace, timing, answer, request, question, tokenizer, + metadata, backend, report.get("noise_values"), reference, + atol, rtol)) + except (ValueError, TypeError, KeyError, AttributeError) as exc: + item["reason"] = str(exc) + failed = any(item["status"] != "passed" for item in result["questions"]) + result.update(status="failed" if failed else "passed", numerical_comparison=True, + contract_error_verified=False) + except (ValueError, TypeError, KeyError, AttributeError) as exc: + result.update(status="failed", reason=str(exc), numerical_comparison=False, + contract_error_verified=False) + counts = Counter(item["status"] for item in results) + numeric = Counter(item["status"] for row in results for item in row["questions"]) + return {"records": results, + "counts": {key: counts[key] for key in + ("passed", "failed", "unverified", "contract_error_verified")}, + "numerical_questions": {key: numeric[key] for key in ("passed", "failed")}, + "all_numerical_comparisons_passed": numeric["passed"] > 0 and not numeric["failed"] + and not counts["failed"], + "complete_contract_verified": not counts["failed"] and not counts["unverified"], + "calibration": None, + "postprocess": {"temperature": 1.0, "abstain_threshold": 0.55, + "answerability_threshold": 0.5}, + "limitations": [ + "The browser path uses uncalibrated probabilities and default thresholds.", + "Unsupported score questions do not count as numerical or contract passes.", + "Provider execution partition, lifecycle, latency and offline operation are not verified.", + "Canonical request hashes support lowercase ASCII object keys and safe integer numbers; Unicode string values are supported.", + "This reference checks the supplied full graph, not shared-bundle SDK execution or task accuracy.", + ]} + + +def verify_report(report_path: Path, manifest_path: Path, output_path: Path, + atol: float = 1e-5, rtol: float = 1e-5) -> dict[str, Any]: + if output_path.exists(): + raise FileExistsError(f"refusing to overwrite evidence: {output_path}") + from tokenizers import Tokenizer + + report = read_json(report_path) + artifacts = validate_artifacts(report, manifest_path) + tokenizer = Tokenizer.from_file(str(artifacts["tokenizer"])) + tokenizer.no_truncation() + tokenizer.no_padding() + reference = CpuReference(artifacts["graph"]) + results = verify_records(report, artifacts["manifest"], tokenizer, reference, atol, rtol) + output = {"schema": "vons.browser-parity-reference/v1", "backend": report["backend"], + "requested_provider": report.get("requested_provider"), + "provider_execution_verified": False, + "manifest_sha256": file_hash(manifest_path), + "report_sha256": file_hash(report_path), "tokenizer_sha256": report["tokenizer_hash"], + "verifier_sha256": file_hash(Path(__file__)), "manifest_verification": artifacts["verification"], + "reference_runtime": {"provider": "CPUExecutionProvider", "ort_version": reference.version, + "intra_op_threads": 1, "inter_op_threads": 1}, + "tolerances": {"atol": atol, "rtol": rtol, "equal_nan": False}, **results} + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("x", encoding="utf-8") as handle: + json.dump(output, handle, indent=2, ensure_ascii=False, allow_nan=False) + handle.write("\n") + return output + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report", required=True, type=Path) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--atol", type=float, default=1e-5) + parser.add_argument("--rtol", type=float, default=1e-5) + args = parser.parse_args() + try: + result = verify_report(args.report, args.manifest, args.output, args.atol, args.rtol) + except (ValueError, OSError, TypeError, KeyError) as exc: + parser.exit(2, f"parity verification rejected: {exc}\n") + print(json.dumps({"output": str(args.output), "counts": result["counts"], + "numerical_questions": result["numerical_questions"]}, allow_nan=False)) + return 0 if result["all_numerical_comparisons_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verify_onnx_bundle.py b/tools/verify_onnx_bundle.py new file mode 100644 index 0000000000000000000000000000000000000000..2bed553bd8b2a173b75c5356dc904f8c53c2e10a --- /dev/null +++ b/tools/verify_onnx_bundle.py @@ -0,0 +1,113 @@ +"""Compare an exported Direct bundle with its PyTorch checkpoint on fixed inputs.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import onnxruntime as ort +import torch +from transformers import AutoModel, AutoTokenizer + +from vons.models import ModelConfig, build_model + + +def verify(checkpoint_path: Path, bundle_path: Path, *, max_length: int = 32) -> dict[str, object]: + checkpoint = torch.load(checkpoint_path, map_location="cpu") + if checkpoint["backend"] not in {"direct", "diffusion"}: + raise ValueError("unsupported checkpoint backend") + encoder_config = checkpoint["config"]["encoder"] + tokenizer = AutoTokenizer.from_pretrained(encoder_config["name"], revision=encoder_config.get("revision")) + encoder = AutoModel.from_pretrained(encoder_config["name"], revision=encoder_config.get("revision")).eval() + backend = checkpoint["backend"] + head = build_model( + backend, + option_count=checkpoint["option_count"], + config=ModelConfig( + hidden_size=checkpoint["hidden_size"], + diffusion_steps=int(checkpoint["config"].get("diffusion_steps", 32)), + inference_steps=int(checkpoint["config"].get("inference_steps", 4)), + ), + ) + head.load_state_dict(checkpoint["state_dict"]) + head.eval() + texts = [ + "The user asked for weather in Seoul.\nQuestion: Which action?\nCandidate: call_weather", + "The user asked for weather in Seoul.\nQuestion: Which action?\nCandidate: clarify", + ] + token_length = max_length if backend == "direct" else int(encoder_config.get("max_tokens", 512)) + encoded = tokenizer(texts, padding="max_length", truncation=True, max_length=token_length, return_tensors="pt") + with torch.no_grad(): + candidate_embeddings = encoder(**encoded).last_hidden_state[:, 0, :].unsqueeze(0) + option_mask = torch.zeros((1, checkpoint["option_count"]), dtype=torch.bool) + if backend == "direct": + candidate_embeddings = candidate_embeddings[:, :2] + option_mask = torch.ones((1, 2), dtype=torch.bool) + pooled = candidate_embeddings.mean(dim=1) + torch_output = head(candidate_embeddings, pooled, option_mask) + else: + padding = checkpoint["option_count"] - candidate_embeddings.shape[1] + candidate_embeddings = torch.cat((candidate_embeddings, torch.zeros((1, padding, candidate_embeddings.shape[-1]))), dim=1) + option_mask[:, :2] = True + pooled = candidate_embeddings[:, :2].mean(dim=1) + betas = torch.linspace(1e-4, 0.02, int(checkpoint["config"].get("diffusion_steps", 32))) + alpha_bars = torch.cumprod(1.0 - betas, dim=0) + timesteps = torch.linspace(len(alpha_bars) - 1, 0, int(checkpoint["config"].get("inference_steps", 4))).round().long() + sample = torch.zeros((1, checkpoint["option_count"])) + for index, timestep in enumerate(timesteps): + predicted_noise = head(sample, pooled, torch.full((1,), int(timestep), dtype=torch.long))["noise"] + alpha = alpha_bars[timestep] + previous_alpha = alpha_bars[timesteps[index + 1]] if index + 1 < len(timesteps) else sample.new_tensor(1.0) + predicted_x0 = (sample - torch.sqrt(1 - alpha) * predicted_noise) / torch.sqrt(alpha) + sample = torch.sqrt(previous_alpha) * predicted_x0 + torch.sqrt(1 - previous_alpha) * predicted_noise + torch_output = { + "scores": sample.masked_fill(~option_mask, float("-inf")), + "answerability": head(torch.zeros_like(sample), pooled, torch.zeros((1,), dtype=torch.long))["answerability"], + } + session = ort.InferenceSession(str(bundle_path), providers=["CPUExecutionProvider"]) + options = checkpoint["option_count"] if backend == "diffusion" else 2 + ort_feed = { + "input_ids": encoded["input_ids"].numpy()[None, :, :] if backend == "direct" else np.pad(encoded["input_ids"].numpy()[None, :, :], ((0, 0), (0, options - 2), (0, 0))), + "attention_mask": encoded["attention_mask"].numpy()[None, :, :] if backend == "direct" else np.pad(encoded["attention_mask"].numpy()[None, :, :], ((0, 0), (0, options - 2), (0, 0))), + "token_type_ids": encoded["token_type_ids"].numpy()[None, :, :] if backend == "direct" else np.pad(encoded["token_type_ids"].numpy()[None, :, :], ((0, 0), (0, options - 2), (0, 0))), + "option_mask": np.asarray([[True, True] + [False] * (options - 2)], dtype=np.bool_), + } + if backend == "diffusion": + ort_feed["initial_noise"] = np.zeros((1, options), dtype=np.float32) + ort_output = session.run(None, ort_feed) + score_name = "logits" if backend == "direct" else "scores" + score_delta = torch_output[score_name].numpy() - ort_output[0] + finite_scores = np.isfinite(torch_output[score_name].numpy()) & np.isfinite(ort_output[0]) + max_score_delta = float(np.max(np.abs(score_delta[finite_scores]))) if finite_scores.any() else 0.0 + result = { + "checkpoint": str(checkpoint_path), + "bundle": str(bundle_path), + "provider": "CPUExecutionProvider", + "backend": backend, + "max_abs_logits": max_score_delta, + "max_abs_answerability": float(np.max(np.abs(torch_output["answerability"].numpy() - ort_output[1]))), + "pass": bool( + np.allclose(torch_output[score_name].numpy(), ort_output[0], atol=1e-5, rtol=1e-5, equal_nan=True) + and np.allclose(torch_output["answerability"].numpy(), ort_output[1], atol=1e-5, rtol=1e-5) + ), + } + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint", required=True, type=Path) + parser.add_argument("--bundle", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + result = verify(args.checkpoint, args.bundle) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 if result["pass"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vons/__init__.py b/vons/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3b96268a8f0d366679de5376c741b11c37b52136 --- /dev/null +++ b/vons/__init__.py @@ -0,0 +1,33 @@ +"""Vons public Python API.""" + +from .contract import ( + Backend, + DecisionRequest, + DecisionResponse, + KASIAction, + Question, + QuestionType, + ResponseStatus, + validate_request, +) +from .kasi import KASIActionDecision, KASIAdapter, KASICall, KASIProposal, ToolPolicy +from .mind2web import Mind2WebExample, Mind2WebMetrics, evaluate_mind2web + +__all__ = [ + "Backend", + "DecisionRequest", + "DecisionResponse", + "KASIAction", + "KASIActionDecision", + "KASIAdapter", + "KASICall", + "KASIProposal", + "Mind2WebExample", + "Mind2WebMetrics", + "Question", + "QuestionType", + "ResponseStatus", + "ToolPolicy", + "evaluate_mind2web", + "validate_request", +] diff --git a/vons/bundle.py b/vons/bundle.py new file mode 100644 index 0000000000000000000000000000000000000000..46e1abb90e4e3bf12c9e4f54e3f98ecb5ee82232 --- /dev/null +++ b/vons/bundle.py @@ -0,0 +1,557 @@ +"""Build and verify content-addressed manifests for deployable ONNX bundles. + +The exporter writes a small legacy manifest with semantic metadata, but that +manifest does not prove which bytes a browser will load. This module treats +the ONNX graph, its external-data references, tokenizer files, and explicitly +selected runtime assets as a single bundle contract. Source checkpoints and +other files left in an experiment directory are inventoried but excluded from +the deployable byte total unless explicitly selected as a runtime asset. +""" + +from __future__ import annotations + +import hashlib +import json +import platform +import subprocess +import sys +from collections.abc import Iterable, Mapping, Sequence +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any + +SCHEMA_VERSION = "vons.bundle.manifest/v1" +MODEL_ASSET_ROLES = frozenset({"model_graph", "model_external_data", "tokenizer", "calibration"}) +DISTRIBUTION_ROLES = MODEL_ASSET_ROLES | frozenset({"runtime_asset", "license_notice", "config"}) + + +class BundleManifestError(ValueError): + """Raised when a bundle cannot be safely enumerated or verified.""" + + +def _require_onnx() -> Any: + try: + import onnx # type: ignore[import-not-found] + except ImportError as exc: # pragma: no cover - exercised in minimal installs + raise RuntimeError("bundle manifest generation requires the optional 'onnx' package") from exc + return onnx + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _json_bytes(value: Mapping[str, Any]) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8") + + +def _root_path(root: Path) -> Path: + resolved = root.resolve(strict=True) + if not resolved.is_dir(): + raise BundleManifestError(f"bundle root is not a directory: {root}") + return resolved + + +def _safe_relative(root: Path, value: str | Path, *, label: str) -> Path: + """Resolve a bundle-relative path and reject traversal and symlinks. + + ONNX external-data locations are specification strings, not arbitrary local + paths. Rejecting both POSIX and Windows absolute forms avoids accepting a + Windows drive path when a manifest is generated on another platform. + """ + + raw = str(value).replace("\\", "/") + posix = PurePosixPath(raw) + windows = PureWindowsPath(raw) + if posix.is_absolute() or windows.is_absolute() or windows.drive: + raise BundleManifestError(f"{label} must be relative to the bundle: {value!r}") + if any(part == ".." for part in posix.parts): + raise BundleManifestError(f"{label} contains traversal: {value!r}") + raw_candidate = root / Path(*posix.parts) + if raw_candidate.is_symlink(): + raise BundleManifestError(f"{label} may not be a symlink: {value!r}") + candidate = raw_candidate.resolve(strict=False) + try: + candidate.relative_to(root) + except ValueError as exc: + raise BundleManifestError(f"{label} escapes the bundle root: {value!r}") from exc + return candidate + + +def _bundle_input(root: Path, value: str | Path, *, label: str) -> Path: + """Accept a caller's absolute path only when it is inside the bundle.""" + + candidate = Path(value) + if candidate.is_absolute() or PureWindowsPath(str(value)).drive: + try: + relative = candidate.resolve(strict=False).relative_to(root) + except ValueError as exc: + raise BundleManifestError(f"{label} is outside the bundle root: {value!r}") from exc + return _safe_relative(root, relative, label=label) + caller_relative = (Path.cwd() / candidate).resolve(strict=False) + if caller_relative.exists(): + try: + relative = caller_relative.relative_to(root) + except ValueError as exc: + raise BundleManifestError(f"{label} is outside the bundle root: {value!r}") from exc + return _safe_relative(root, relative, label=label) + return _safe_relative(root, value, label=label) + + +def _relative(root: Path, path: Path) -> str: + try: + return path.resolve(strict=True).relative_to(root).as_posix() + except ValueError as exc: + raise BundleManifestError(f"path is outside bundle root: {path}") from exc + + +def _regular_files(root: Path, directory: Path) -> list[Path]: + if directory.is_symlink(): + raise BundleManifestError(f"bundle directory may not be a symlink: {directory}") + if not directory.is_dir(): + raise BundleManifestError(f"expected directory: {directory}") + files: list[Path] = [] + for path in sorted(directory.rglob("*")): + if path.is_symlink(): + raise BundleManifestError(f"bundle file may not be a symlink: {path}") + if path.is_file(): + _safe_relative(root, path.relative_to(root), label="bundle file") + files.append(path) + return files + + +def _external_locations(model: Any, onnx: Any) -> set[str]: + """Return external-data locations from initializers and nested subgraphs.""" + + locations: set[str] = set() + + def visit_tensor(tensor: Any) -> None: + for item in tensor.external_data: + if item.key == "location": + if not item.value: + raise BundleManifestError("ONNX external_data has an empty location") + locations.add(item.value) + + def visit_graph(graph: Any) -> None: + for tensor in graph.initializer: + visit_tensor(tensor) + for sparse in getattr(graph, "sparse_initializer", []): + visit_tensor(sparse.values) + visit_tensor(sparse.indices) + for node in graph.node: + for attribute in node.attribute: + if attribute.type == onnx.AttributeProto.GRAPH: + visit_graph(attribute.g) + elif attribute.type == onnx.AttributeProto.GRAPHS: + for nested in attribute.graphs: + visit_graph(nested) + elif attribute.type == onnx.AttributeProto.TENSOR: + visit_tensor(attribute.t) + elif attribute.type == onnx.AttributeProto.TENSORS: + for tensor in attribute.tensors: + visit_tensor(tensor) + elif attribute.type == onnx.AttributeProto.SPARSE_TENSOR: + visit_tensor(attribute.sparse_tensor.values) + visit_tensor(attribute.sparse_tensor.indices) + elif attribute.type == onnx.AttributeProto.SPARSE_TENSORS: + for sparse in attribute.sparse_tensors: + visit_tensor(sparse.values) + visit_tensor(sparse.indices) + + visit_graph(model.graph) + return locations + + +def _shape(value_info: Any) -> list[int | str | None] | None: + tensor_type = value_info.type.tensor_type + if not tensor_type.HasField("shape"): + return None + result: list[int | str | None] = [] + for dimension in tensor_type.shape.dim: + if dimension.HasField("dim_value"): + result.append(int(dimension.dim_value)) + elif dimension.HasField("dim_param"): + result.append(dimension.dim_param) + else: + result.append(None) + return result + + +def _value_infos(values: Iterable[Any], onnx: Any) -> list[dict[str, Any]]: + result = [] + for value in values: + tensor_type = value.type.tensor_type + result.append( + { + "name": value.name, + "dtype": onnx.TensorProto.DataType.Name(tensor_type.elem_type), + "shape": _shape(value), + } + ) + return result + + +def _graph_metadata(model: Any, onnx: Any, external_locations: Sequence[str]) -> dict[str, Any]: + return { + "ir_version": int(model.ir_version), + "opset_imports": [ + {"domain": item.domain, "version": int(item.version)} for item in model.opset_import + ], + "inputs": _value_infos(model.graph.input, onnx), + "outputs": _value_infos(model.graph.output, onnx), + "external_data_locations": sorted(external_locations), + "node_count": len(model.graph.node), + } + + +def _git_revision(root: Path) -> dict[str, str | None]: + try: + value = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + capture_output=True, + check=True, + text=True, + timeout=2, + ).stdout.strip() + except (OSError, subprocess.SubprocessError): + value = "" + return {"value": value or None, "reason": None if value else "not a git checkout or revision unavailable"} + + +def _snapshot_hash( + root: Path | None, + paths: Sequence[str | Path] | None = None, +) -> dict[str, Any]: + if root is None: + return { + "sha256": None, + "reason": "source root was not supplied", + "scope": None, + "paths": [], + } + source_root = root.resolve(strict=True) + if not source_root.is_dir(): + raise BundleManifestError(f"source root is not a directory: {root}") + digest = hashlib.sha256() + ignored_directories = {".git", "__pycache__", ".pytest_cache", ".ruff_cache"} + files: list[tuple[Path, Path]] = [] + if paths: + for supplied in paths: + candidate = Path(supplied) + if not candidate.is_absolute(): + candidate = source_root / candidate + if candidate.is_symlink(): + raise BundleManifestError(f"source snapshot path may not be a symlink: {supplied}") + resolved = candidate.resolve(strict=True) + try: + relative_path = resolved.relative_to(source_root) + except ValueError as exc: + raise BundleManifestError( + f"source snapshot path escapes the source root: {supplied}" + ) from exc + if not resolved.is_file(): + raise BundleManifestError(f"source snapshot path is not a file: {supplied}") + if any(part in ignored_directories for part in relative_path.parts) or resolved.suffix == ".pyc": + raise BundleManifestError(f"source snapshot path is ignored: {supplied}") + files.append((relative_path, resolved)) + if len({relative for relative, _ in files}) != len(files): + raise BundleManifestError("source snapshot paths must be unique") + scope = "explicit" + else: + for path in source_root.rglob("*"): + if not path.is_file() or path.is_symlink() or path.suffix == ".pyc": + continue + relative_path = path.relative_to(source_root) + if any(part in ignored_directories for part in relative_path.parts): + continue + files.append((relative_path, path)) + scope = "all_eligible_files" + for relative_path, path in sorted(files): + relative = relative_path.as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(path.stat().st_size.to_bytes(8, "big")) + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return { + "sha256": digest.hexdigest(), + "reason": None, + "scope": scope, + "paths": [relative.as_posix() for relative, _ in sorted(files)], + } + + +def _legacy_metadata(root: Path, output: Path) -> dict[str, Any]: + candidate = root / "bundle-manifest.json" + if candidate == output or not candidate.is_file(): + return {} + try: + value = json.loads(candidate.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return value if isinstance(value, dict) else {} + + +def _file_record(root: Path, path: Path, role: str, *, runtime_loaded: bool, model_asset: bool) -> dict[str, Any]: + relative = _relative(root, path) + size = path.stat().st_size + return { + "path": relative, + "role": role, + "bytes": size, + "sha256": _sha256(path), + "compression": {"encoding": "identity", "measured": False, "bytes": size}, + "runtime_loaded": runtime_loaded, + "model_asset": model_asset, + } + + +def _stable_manifest(manifest: dict[str, Any], output: Path, entries: Sequence[dict[str, Any]]) -> None: + """Write the manifest until its reported own size reaches a fixed point.""" + + payload_bytes = sum(int(item["bytes"]) for item in entries if item["model_asset"]) + distribution_bytes = sum(int(item["bytes"]) for item in entries if item["role"] in DISTRIBUTION_ROLES) + manifest["summary"]["model_asset_bytes"] = payload_bytes + manifest["summary"]["manifest_bytes"] = 0 + manifest["summary"]["complete_download_bytes"] = distribution_bytes + output.parent.mkdir(parents=True, exist_ok=True) + for _ in range(8): + output.write_bytes(_json_bytes(manifest)) + manifest_bytes = output.stat().st_size + complete = distribution_bytes + manifest_bytes + if ( + manifest["summary"]["manifest_bytes"] == manifest_bytes + and manifest["summary"]["complete_download_bytes"] == complete + ): + return + manifest["summary"]["manifest_bytes"] = manifest_bytes + manifest["summary"]["complete_download_bytes"] = complete + raise BundleManifestError("manifest size did not converge") + + +def build_bundle_manifest( + onnx_path: str | Path, + tokenizer_path: str | Path, + output_path: str | Path, + *, + checkpoint_path: str | Path | None = None, + model_manifest_path: str | Path | None = None, + calibration_paths: Sequence[str | Path] = (), + runtime_asset_paths: Sequence[str | Path] = (), + license_paths: Sequence[str | Path] = (), + config_paths: Sequence[str | Path] = (), + source_root: str | Path | None = None, + source_paths: Sequence[str | Path] = (), +) -> Path: + """Create a deployable manifest and inventory every regular file in its root.""" + + output = Path(output_path).resolve(strict=False) + root = _root_path(output.parent) + tokenizer = _bundle_input(root, tokenizer_path, label="tokenizer path") + graph_path = _bundle_input(root, onnx_path, label="ONNX path") + if not graph_path.is_file(): + raise BundleManifestError(f"ONNX path is not a file: {onnx_path}") + if output.parent != root: + raise BundleManifestError("manifest output must be directly inside the bundle root") + + onnx = _require_onnx() + try: + model = onnx.load_model(str(graph_path), load_external_data=False) + except Exception as exc: # onnx raises several protobuf-specific exceptions + raise BundleManifestError(f"could not parse ONNX graph {graph_path}: {exc}") from exc + external_locations = _external_locations(model, onnx) + external_files: dict[Path, str] = {} + for location in sorted(external_locations): + path = _safe_relative(root, location, label="ONNX external-data location") + if not path.is_file(): + raise BundleManifestError(f"ONNX external-data file does not exist: {location}") + external_files[path] = location + + paths_by_role: dict[Path, str] = {graph_path: "model_graph"} + if tokenizer.is_file(): + paths_by_role[tokenizer] = "tokenizer" + else: + tokenizer_files = _regular_files(root, tokenizer) + if not tokenizer_files: + raise BundleManifestError("tokenizer directory contains no regular files") + for path in tokenizer_files: + paths_by_role[path] = "tokenizer" + for path in external_files: + paths_by_role[path] = "model_external_data" + + explicit_groups = [ + (checkpoint_path, "source_checkpoint"), + (model_manifest_path, "source_manifest"), + ] + for values, role in ( + (calibration_paths, "calibration"), + (runtime_asset_paths, "runtime_asset"), + (license_paths, "license_notice"), + (config_paths, "config"), + ): + explicit_groups.extend((value, role) for value in values) + for value, role in explicit_groups: + if value is None: + continue + path = _bundle_input(root, value, label=f"{role} path") + if not path.is_file(): + raise BundleManifestError(f"{role} path is not a file: {value}") + paths_by_role[path] = role + + if checkpoint_path is None: + candidate = root / "model.pt" + if candidate.is_file(): + paths_by_role.setdefault(candidate, "source_checkpoint") + if model_manifest_path is None: + candidate = root / "model-manifest.json" + if candidate.is_file(): + paths_by_role.setdefault(candidate, "source_manifest") + + all_files = _regular_files(root, root) + records: list[dict[str, Any]] = [] + for path in all_files: + if path == output: + continue + role = paths_by_role.get(path) + if role is None: + if path.suffix == ".onnx": + role = "unselected_model_artifact" + elif path.suffix == ".pt": + role = "source_checkpoint" + elif path.name.endswith(".onnx.data"): + role = "unreferenced_model_artifact" + elif path.name.endswith("manifest.json"): + role = "source_manifest" + else: + role = "extra" + records.append( + _file_record( + root, + path, + role, + runtime_loaded=role in {"model_graph", "model_external_data", "tokenizer", "runtime_asset"}, + model_asset=role in MODEL_ASSET_ROLES, + ) + ) + records.sort(key=lambda item: item["path"]) + if not any(item["role"] == "model_graph" for item in records): + raise BundleManifestError("manifest inventory does not contain the selected ONNX graph") + + legacy = _legacy_metadata(root, output) + metadata = {key: value for key, value in legacy.items() if key not in {"onnx", "tokenizer", "checkpoint"}} + metadata["graph"] = _graph_metadata(model, onnx, sorted(external_locations)) + metadata["checkpoint_sha256"] = next( + (item["sha256"] for item in records if item["role"] == "source_checkpoint"), None + ) + manifest = { + "schema_version": SCHEMA_VERSION, + "manifest": { + "path": output.name, + "sha256": None, + "self_hash_excluded": True, + }, + "bundle_root": ".", + "metadata": metadata, + "source": { + "git_revision": _git_revision(root), + "source_snapshot": _snapshot_hash( + Path(source_root) if source_root is not None else None, + source_paths or None, + ), + "builder": {"python": sys.version.split()[0], "platform": platform.platform()}, + }, + "files": records, + "summary": { + "model_asset_bytes": 0, + "manifest_bytes": 0, + "complete_download_bytes": 0, + "model_asset_limit_bytes": 67_108_864, + "within_model_asset_limit": False, + "runtime_loaded_file_count": sum(item["runtime_loaded"] for item in records), + }, + } + _stable_manifest(manifest, output, records) + manifest["summary"]["within_model_asset_limit"] = ( + manifest["summary"]["model_asset_bytes"] <= manifest["summary"]["model_asset_limit_bytes"] + ) + output.write_bytes(_json_bytes(manifest)) + # Updating the boolean can alter the byte count; converge once more without + # including the manifest in its own digest or model-asset total. + _stable_manifest(manifest, output, records) + return output + + +def verify_bundle_manifest(manifest_path: str | Path) -> dict[str, Any]: + """Verify every recorded file, graph reference, and byte summary.""" + + manifest_file = Path(manifest_path).resolve(strict=True) + payload = json.loads(manifest_file.read_text(encoding="utf-8")) + if payload.get("schema_version") != SCHEMA_VERSION: + raise BundleManifestError(f"unsupported bundle manifest schema: {payload.get('schema_version')!r}") + root = manifest_file.parent + manifest_info = payload.get("manifest", {}) + if manifest_info.get("path") != manifest_file.name: + raise BundleManifestError("manifest path does not match the verification target") + records = payload.get("files") + if not isinstance(records, list) or not records: + raise BundleManifestError("manifest has no file inventory") + seen: set[str] = set() + for item in records: + if not isinstance(item, dict): + raise BundleManifestError("manifest file entry is not an object") + relative = item.get("path") + if not isinstance(relative, str) or relative in seen: + raise BundleManifestError(f"invalid or duplicate manifest path: {relative!r}") + seen.add(relative) + path = _safe_relative(root, relative, label="manifest file path") + if not path.is_file(): + raise BundleManifestError(f"manifest file is missing: {relative}") + actual_bytes = path.stat().st_size + actual_hash = _sha256(path) + if actual_bytes != item.get("bytes") or actual_hash != item.get("sha256"): + raise BundleManifestError(f"manifest file changed: {relative}") + + graph_item = next((item for item in records if item.get("role") == "model_graph"), None) + if graph_item is None: + raise BundleManifestError("manifest has no model_graph entry") + onnx = _require_onnx() + model = onnx.load_model(str(root / graph_item["path"]), load_external_data=False) + external_locations = _external_locations(model, onnx) + inventory_paths = {item["path"] for item in records} + for location in external_locations: + path = _safe_relative(root, location, label="ONNX external-data location") + if _relative(root, path) not in inventory_paths: + raise BundleManifestError(f"external-data file is not in inventory: {location}") + + summary = payload.get("summary", {}) + model_asset_bytes = sum(item["bytes"] for item in records if item.get("model_asset")) + distribution_bytes = sum(item["bytes"] for item in records if item.get("role") in DISTRIBUTION_ROLES) + if summary.get("model_asset_bytes") != model_asset_bytes: + raise BundleManifestError("model_asset_bytes does not match the inventory") + if summary.get("manifest_bytes") != manifest_file.stat().st_size: + raise BundleManifestError("manifest_bytes does not match the manifest file") + if summary.get("complete_download_bytes") != distribution_bytes + manifest_file.stat().st_size: + raise BundleManifestError("complete_download_bytes does not match the inventory") + if summary.get("within_model_asset_limit") != ( + model_asset_bytes <= int(summary.get("model_asset_limit_bytes", 67_108_864)) + ): + raise BundleManifestError("within_model_asset_limit is stale") + return { + "manifest": str(manifest_file), + "files": len(records), + "external_data_files": len(external_locations), + "model_asset_bytes": model_asset_bytes, + "complete_download_bytes": distribution_bytes + manifest_file.stat().st_size, + "pass": True, + } + + +def build_and_verify(*args: Any, **kwargs: Any) -> tuple[Path, dict[str, Any]]: + """Build a manifest and immediately run the independent verifier.""" + + output = build_bundle_manifest(*args, **kwargs) + return output, verify_bundle_manifest(output) diff --git a/vons/calibration.py b/vons/calibration.py new file mode 100644 index 0000000000000000000000000000000000000000..24837ec344bb0eeaa4ec8f5fea54c5da002d4783 --- /dev/null +++ b/vons/calibration.py @@ -0,0 +1,491 @@ +"""Post-hoc probability and abstention calibration for saved Vons reports. + +Calibration contract ``vons.calibration/v2``: + +- Probability vectors must be finite, non-negative, and normalized. Options + with true zero probability stay at zero under temperature scaling. +- Saved predictions follow the producer contract + ``confidence = max_probability * answerability``. The answerability factor + is reconstructed only when ``0 <= confidence <= max_probability`` (within + ``CONFIDENCE_FACTOR_TOLERANCE``); any other confidence fails closed. +- A prediction is eligible for coverage only when it has no error, carries a + probability vector, and its answerability meets the configured threshold + (report config ``answerability_threshold``, default 0.5). Ground-truth labels + never gate eligibility; they are used only to measure fitting risk. +- An eligible prediction is covered when its calibrated confidence meets the + abstention threshold; its choice is then the argmax of the calibrated + vector. Every other prediction abstains with ``choice=None``, and a failed + prediction keeps its error. +- ``ABSTAIN_ALL_THRESHOLD`` (2.0) is the documented sentinel for "abstain on + every prediction". Confidences are validated in [0, 1], so the sentinel + covers nothing, including wrong predictions with confidence 1.0. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Any + +from .data import Example, read_jsonl +from .evaluation import ( + NLL_PROBABILITY_FLOOR, + Prediction, + summarize, + validate_distribution, + validate_predictions, +) + +CALIBRATION_VERSION = "vons.calibration/v2" +ABSTAIN_ALL_THRESHOLD = 2.0 +DEFAULT_ANSWERABILITY_THRESHOLD = 0.5 +CONFIDENCE_FACTOR_TOLERANCE = 1e-6 +DECISION_EPSILON = 1e-12 +BOUND_TOLERANCE = 1e-6 +IDENTITY_FIELDS = ("backend", "model_id", "encoder", "seed", "checkpoint", "checkpoint_sha256") + + +@dataclass(frozen=True) +class CalibrationFit: + """Parameters fitted on a calibration split and then reused unchanged. + + ``rows`` counts every calibration example. ``temperature_fit_rows`` counts + the answerable rows with a usable probability vector, and + ``threshold_fit_rows`` counts the rows with a prediction. + """ + + temperature: float + abstention_threshold: float + target_risk: float + rows: int + nll_before: float | None + nll_after: float | None + temperature_fit_rows: int = 0 + threshold_fit_rows: int = 0 + threshold_covered_rows: int = 0 + threshold_risk: float | None = None + temperature_lower_bound: float = 0.05 + temperature_upper_bound: float = 20.0 + temperature_at_lower_bound: bool = False + temperature_at_upper_bound: bool = False + abstain_all: bool = False + answerability_threshold: float = DEFAULT_ANSWERABILITY_THRESHOLD + version: str = CALIBRATION_VERSION + + +@dataclass(frozen=True) +class TemperatureFit: + temperature: float + rows: int + lower: float + upper: float + at_lower_bound: bool + at_upper_bound: bool + + +@dataclass(frozen=True) +class ThresholdFit: + threshold: float + rows: int + covered_rows: int + risk: float | None + abstain_all: bool + + +@dataclass(frozen=True) +class _Decision: + probabilities: Mapping[str, float] + confidence: float + choice: str | None + eligible: bool + + +def _validate_temperature(temperature: float) -> float: + value = float(temperature) + if not math.isfinite(value) or value <= 0: + raise ValueError("temperature must be a finite positive number") + return value + + +def _validate_unit_interval(value: float, name: str) -> float: + number = float(value) + if not math.isfinite(number) or not 0.0 <= number <= 1.0: + raise ValueError(f"{name} must be finite and in [0, 1]") + return number + + +def _validate_abstention_threshold(value: float) -> float: + number = float(value) + if number == ABSTAIN_ALL_THRESHOLD: + return number + if not math.isfinite(number) or not 0.0 <= number <= 1.0: + raise ValueError("abstention_threshold must be in [0, 1] or equal ABSTAIN_ALL_THRESHOLD") + return number + + +def _scaled_probabilities(probabilities: Mapping[str, float], temperature: float) -> dict[str, float]: + """Temperature-scale a validated vector; true zeros stay zero.""" + _validate_temperature(temperature) + values = validate_distribution(probabilities) + if not values: + return {} + logs = {option: math.log(value) / temperature for option, value in values.items() if value > 0.0} + pivot = max(logs.values()) + weights = {option: math.exp(value - pivot) for option, value in logs.items()} + total = math.fsum(weights.values()) + return {option: weights[option] / total if option in weights else 0.0 for option in values} + + +def _argmax(probabilities: Mapping[str, float], options: Sequence[str]) -> str: + best = max(probabilities.get(option, 0.0) for option in options) + return next(option for option in options if probabilities.get(option, 0.0) == best) + + +def _answerability_factor(prediction: Prediction, distribution: Mapping[str, float]) -> float: + maximum = max(distribution.values()) + confidence = float(prediction.confidence) + if not -CONFIDENCE_FACTOR_TOLERANCE <= confidence <= maximum + CONFIDENCE_FACTOR_TOLERANCE: + raise ValueError( + f"prediction {prediction.example_id!r} violates confidence = max_probability * answerability " + f"(confidence={confidence!r}, max_probability={maximum!r})" + ) + return min(1.0, max(0.0, confidence) / maximum) + + +def _decide(prediction: Prediction, row: Example, *, temperature: float, answerability_threshold: float) -> _Decision: + if prediction.error is not None: + return _Decision(prediction.probabilities, float(prediction.confidence), None, False) + distribution = validate_distribution(prediction.probabilities, row.options) + if not distribution: + return _Decision({}, float(prediction.confidence), None, False) + factor = _answerability_factor(prediction, distribution) + calibrated = _scaled_probabilities(distribution, temperature) + confidence = min(1.0, max(0.0, factor * max(calibrated.values()))) + eligible = factor + DECISION_EPSILON >= answerability_threshold + return _Decision(calibrated, confidence, _argmax(calibrated, row.options), eligible) + + +def _is_covered(decision: _Decision, threshold: float) -> bool: + return decision.eligible and decision.confidence + DECISION_EPSILON >= threshold + + +def _temperature_rows( + rows: Mapping[str, Example], + lookup: Mapping[str, Prediction], +) -> list[tuple[Example, dict[str, float]]]: + usable: list[tuple[Example, dict[str, float]]] = [] + for row in rows.values(): + item = lookup.get(row.id) + if not row.answerable or row.label is None or item is None or item.error is not None: + continue + distribution = validate_distribution(item.probabilities, row.options) + if distribution: + usable.append((row, distribution)) + return usable + + +def _mean_nll(usable: Sequence[tuple[Example, Mapping[str, float]]], temperature: float) -> float: + total = math.fsum( + -math.log(max(_scaled_probabilities(distribution, temperature).get(str(row.label), 0.0), NLL_PROBABILITY_FLOOR)) + for row, distribution in usable + ) + return total / len(usable) + + +def _nll_at(usable: Sequence[tuple[Example, Mapping[str, float]]], temperature: float) -> float | None: + return _mean_nll(usable, temperature) if usable else None + + +def fit_temperature_details( + examples: Sequence[Example], + predictions: Sequence[Prediction], + *, + lower: float = 0.05, + upper: float = 20.0, + iterations: int = 80, +) -> TemperatureFit: + """Fit a scalar temperature by deterministic golden-section search on log T.""" + if not (math.isfinite(lower) and math.isfinite(upper)) or lower <= 0 or upper <= lower: + raise ValueError("temperature bounds must satisfy 0 < lower < upper") + rows, lookup = validate_predictions(examples, predictions) + usable = _temperature_rows(rows, lookup) + if not usable: + return TemperatureFit(1.0, 0, lower, upper, False, False) + + def objective(log_temperature: float) -> float: + return _mean_nll(usable, math.exp(log_temperature)) + + left, right = math.log(lower), math.log(upper) + golden = (math.sqrt(5.0) - 1.0) / 2.0 + first = right - golden * (right - left) + second = left + golden * (right - left) + first_value, second_value = objective(first), objective(second) + for _ in range(max(1, iterations)): + if first_value <= second_value: + right, second, second_value = second, first, first_value + first = right - golden * (right - left) + first_value = objective(first) + else: + left, first, first_value = first, second, second_value + second = left + golden * (right - left) + second_value = objective(second) + log_temperature = (left + right) / 2.0 + return TemperatureFit( + temperature=math.exp(log_temperature), + rows=len(usable), + lower=lower, + upper=upper, + at_lower_bound=log_temperature - math.log(lower) <= BOUND_TOLERANCE, + at_upper_bound=math.log(upper) - log_temperature <= BOUND_TOLERANCE, + ) + + +def fit_temperature( + examples: Sequence[Example], + predictions: Sequence[Prediction], + *, + lower: float = 0.05, + upper: float = 20.0, + iterations: int = 80, +) -> float: + """Return the fitted temperature; see ``fit_temperature_details`` for bound flags.""" + return fit_temperature_details(examples, predictions, lower=lower, upper=upper, iterations=iterations).temperature + + +def fit_abstention_threshold_details( + examples: Sequence[Example], + predictions: Sequence[Prediction], + *, + target_risk: float = 0.05, + temperature: float = 1.0, + answerability_threshold: float = DEFAULT_ANSWERABILITY_THRESHOLD, +) -> ThresholdFit: + """Choose the lowest confidence cutoff with maximum coverage whose risk meets the target. + + Coverage, argmax choices, and eligibility match ``apply_calibration``. A + covered prediction on an unanswerable row counts as an error. When no + cutoff meets the target, the fit returns ``ABSTAIN_ALL_THRESHOLD``. + """ + target = float(target_risk) + if not math.isfinite(target) or not 0.0 <= target <= 1.0: + raise ValueError("target_risk must be between 0 and 1") + _validate_temperature(temperature) + _validate_unit_interval(answerability_threshold, "answerability_threshold") + rows, lookup = validate_predictions(examples, predictions) + decisions = [ + (rows[example_id], _decide(item, rows[example_id], temperature=temperature, answerability_threshold=answerability_threshold)) + for example_id, item in lookup.items() + ] + eligible = [(row, decision) for row, decision in decisions if decision.eligible] + best: tuple[int, float, float] | None = None + for threshold in sorted({0.0, *(decision.confidence for _, decision in eligible)}): + covered = [(row, decision) for row, decision in eligible if _is_covered(decision, threshold)] + if not covered: + continue + errors = sum(not row.answerable or decision.choice != row.label for row, decision in covered) + risk = errors / len(covered) + if risk <= target and (best is None or len(covered) > best[0]): + best = (len(covered), threshold, risk) + if best is None: + return ThresholdFit(ABSTAIN_ALL_THRESHOLD, len(decisions), 0, None, True) + return ThresholdFit(best[1], len(decisions), best[0], best[2], False) + + +def fit_abstention_threshold( + examples: Sequence[Example], + predictions: Sequence[Prediction], + *, + target_risk: float = 0.05, + temperature: float = 1.0, + answerability_threshold: float = DEFAULT_ANSWERABILITY_THRESHOLD, +) -> float: + """Return the fitted threshold, or ``ABSTAIN_ALL_THRESHOLD`` when nothing can be covered.""" + return fit_abstention_threshold_details( + examples, + predictions, + target_risk=target_risk, + temperature=temperature, + answerability_threshold=answerability_threshold, + ).threshold + + +def apply_calibration( + examples: Sequence[Example], + predictions: Sequence[Prediction], + *, + temperature: float, + abstention_threshold: float, + answerability_threshold: float = DEFAULT_ANSWERABILITY_THRESHOLD, +) -> list[Prediction]: + """Apply fitted parameters without changing the original prediction objects. + + Covered outputs always carry the argmax choice of the calibrated vector; + every other output abstains with ``choice=None``. + """ + _validate_temperature(temperature) + threshold = _validate_abstention_threshold(abstention_threshold) + _validate_unit_interval(answerability_threshold, "answerability_threshold") + rows, _lookup = validate_predictions(examples, predictions) + calibrated: list[Prediction] = [] + for prediction in predictions: + if prediction.error is not None: + calibrated.append(replace(prediction, choice=None, abstained=True)) + continue + decision = _decide( + prediction, + rows[prediction.example_id], + temperature=temperature, + answerability_threshold=answerability_threshold, + ) + covered = _is_covered(decision, threshold) + calibrated.append( + replace( + prediction, + choice=decision.choice if covered else None, + probabilities=dict(decision.probabilities), + confidence=decision.confidence, + abstained=not covered, + ) + ) + return calibrated + + +def _read_report(path: str | Path) -> tuple[dict[str, Any], list[Prediction]]: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + raise TypeError(f"{path}: report must be a JSON object") + config = payload.get("config", {}) + raw = payload.get("predictions", []) + if not isinstance(config, Mapping) or not isinstance(raw, list): + raise TypeError(f"{path}: report needs a config object and a predictions list") + return dict(config), [Prediction(**item) for item in raw] + + +def _sha256(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _model_identity(calibration_config: Mapping[str, Any], test_config: Mapping[str, Any]) -> dict[str, Any]: + identity: dict[str, Any] = {} + for field in IDENTITY_FIELDS: + left, right = calibration_config.get(field), test_config.get(field) + if left != right: + raise ValueError(f"calibration and test reports disagree on {field}: {left!r} != {right!r}") + identity[field] = left + return identity + + +def _answerability_threshold(calibration_config: Mapping[str, Any], test_config: Mapping[str, Any]) -> float: + values: set[float] = set() + for config in (calibration_config, test_config): + value = config.get("answerability_threshold") + values.add(_validate_unit_interval(DEFAULT_ANSWERABILITY_THRESHOLD if value is None else value, "answerability_threshold")) + if len(values) != 1: + raise ValueError("calibration and test reports use different answerability thresholds") + return values.pop() + + +def _limitations(identity: Mapping[str, Any]) -> list[str]: + limitations: list[str] = [] + if identity.get("checkpoint_sha256") is None: + limitations.append( + "checkpoint_sha256 unavailable: report configs carry no immutable checkpoint hash, so model identity " + "is checked only on backend, model_id, encoder, seed, and checkpoint fields when present" + ) + if identity.get("backend") == "diffusion": + limitations.append( + "N1 (numerics review, unchanged in vons.calibration/v2): Diffusion probabilities are a softmax over " + "x0 estimates, so a perfect one-hot estimate over four options has max probability 0.4754" + ) + limitations.append( + "N2 (numerics review, unchanged in vons.calibration/v2): the 32-step schedule reuses the 1000-step " + "beta range, so terminal alpha_bar is 0.7234 while sampling starts from N(0, I)" + ) + return limitations + + +def calibrate_report( + *, + calibration_input: str | Path, + calibration_report: str | Path, + test_input: str | Path, + test_report: str | Path, + output: str | Path, + target_risk: float = 0.05, +) -> Path: + """Fit on calibration rows and write a calibrated test report as strict JSON.""" + inputs = { + "calibration_input": Path(calibration_input), + "calibration_report": Path(calibration_report), + "test_input": Path(test_input), + "test_report": Path(test_report), + } + target = Path(output) + if target.resolve() in {path.resolve() for path in inputs.values()}: + raise ValueError("output must not overwrite a calibration input") + calibration_examples = read_jsonl(calibration_input) + calibration_config, calibration_predictions = _read_report(calibration_report) + test_examples = read_jsonl(test_input) + test_config, test_predictions = _read_report(test_report) + identity = _model_identity(calibration_config, test_config) + answerability_threshold = _answerability_threshold(calibration_config, test_config) + + temperature_fit = fit_temperature_details(calibration_examples, calibration_predictions) + threshold_fit = fit_abstention_threshold_details( + calibration_examples, + calibration_predictions, + target_risk=target_risk, + temperature=temperature_fit.temperature, + answerability_threshold=answerability_threshold, + ) + calibrated_test = apply_calibration( + test_examples, + test_predictions, + temperature=temperature_fit.temperature, + abstention_threshold=threshold_fit.threshold, + answerability_threshold=answerability_threshold, + ) + usable = _temperature_rows(*validate_predictions(calibration_examples, calibration_predictions)) + fit = CalibrationFit( + temperature=temperature_fit.temperature, + abstention_threshold=threshold_fit.threshold, + target_risk=target_risk, + rows=len(calibration_examples), + nll_before=_nll_at(usable, 1.0), + nll_after=_nll_at(usable, temperature_fit.temperature), + temperature_fit_rows=temperature_fit.rows, + threshold_fit_rows=threshold_fit.rows, + threshold_covered_rows=threshold_fit.covered_rows, + threshold_risk=threshold_fit.risk, + temperature_lower_bound=temperature_fit.lower, + temperature_upper_bound=temperature_fit.upper, + temperature_at_lower_bound=temperature_fit.at_lower_bound, + temperature_at_upper_bound=temperature_fit.at_upper_bound, + abstain_all=threshold_fit.abstain_all, + answerability_threshold=answerability_threshold, + ) + payload = json.dumps( + { + "config": {**test_config, "calibration": asdict(fit)}, + "calibration": asdict(fit), + "inputs": {name: {"path": str(path), "sha256": _sha256(path)} for name, path in inputs.items()}, + "model_identity": identity, + "limitations": _limitations(identity), + "summary": summarize(test_examples, calibrated_test), + "predictions": [asdict(item) for item in calibrated_test], + }, + indent=2, + sort_keys=True, + allow_nan=False, + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(payload, encoding="utf-8") + return target diff --git a/vons/cli.py b/vons/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..3c4af15a5d7bb2c30b20059a3a77098f0290dd80 --- /dev/null +++ b/vons/cli.py @@ -0,0 +1,114 @@ +"""Command-line entry points for reproducible Vons experiments.""" + +from __future__ import annotations + +import argparse +import json +import platform +from pathlib import Path + +from .bundle import build_bundle_manifest, verify_bundle_manifest +from .calibration import calibrate_report +from .data import dataset_manifest, read_jsonl, smoke_examples, synthetic_examples, write_jsonl +from .ollama import OllamaClient, benchmark_ollama, write_benchmark +from .research import build_research_packet +from .training import evaluate, export_diffusion_bundle, export_direct_bundle, export_head, train + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="vons") + sub = parser.add_subparsers(dest="command", required=True) + command = sub.add_parser("generate-smoke"); command.add_argument("--output", required=True) + command = sub.add_parser("generate-synthetic"); command.add_argument("--count", type=int, default=2000); command.add_argument("--seed", type=int, default=7); command.add_argument("--output", required=True) + command = sub.add_parser("validate-data"); command.add_argument("--input", required=True); command.add_argument("--manifest") + command = sub.add_parser("ollama-check"); command.add_argument("--models", nargs="+", required=True); command.add_argument("--host", default="http://127.0.0.1:11434"); command.add_argument("--output") + command = sub.add_parser("ollama-benchmark"); command.add_argument("--model", required=True); command.add_argument("--input", required=True); command.add_argument("--role", default="judge"); command.add_argument("--limit", type=int, default=200); command.add_argument("--seed", type=int, default=7); command.add_argument("--host", default="http://127.0.0.1:11434"); command.add_argument("--output", required=True) + command = sub.add_parser("build-research-packet"); command.add_argument("--results", required=True); command.add_argument("--output", required=True) + command = sub.add_parser("build-bundle-manifest"); command.add_argument("--onnx", required=True); command.add_argument("--tokenizer", required=True); command.add_argument("--output", required=True); command.add_argument("--checkpoint"); command.add_argument("--model-manifest"); command.add_argument("--calibration", action="append", default=[]); command.add_argument("--runtime-asset", action="append", default=[]); command.add_argument("--license", action="append", default=[]); command.add_argument("--config", action="append", default=[]); command.add_argument("--source-root"); command.add_argument("--source-file", action="append", default=[]) + command = sub.add_parser("verify-bundle-manifest"); command.add_argument("--manifest", required=True) + command = sub.add_parser("calibrate"); command.add_argument("--calibration-input", required=True); command.add_argument("--calibration-report", required=True); command.add_argument("--test-input", required=True); command.add_argument("--test-report", required=True); command.add_argument("--output", required=True); command.add_argument("--target-risk", type=float, default=0.05) + for name in ("train", "evaluate", "export", "export-bundle", "export-diffusion-bundle"): + command = sub.add_parser(name); command.add_argument("--config"); command.add_argument("--model"); command.add_argument("--input"); command.add_argument("--checkpoint"); command.add_argument("--output") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + if args.command == "generate-smoke": + write_jsonl(args.output, smoke_examples()); print(json.dumps({"output": args.output, "rows": 4}, indent=2)); return 0 + if args.command == "generate-synthetic": + rows = synthetic_examples(args.count, seed=args.seed); write_jsonl(args.output, rows); print(json.dumps({"output": args.output, "rows": len(rows), "seed": args.seed}, indent=2)); return 0 + if args.command == "validate-data": + rows = read_jsonl(args.input); manifest = dataset_manifest(args.input, rows) + if args.manifest: + Path(args.manifest).parent.mkdir(parents=True, exist_ok=True); Path(args.manifest).write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(manifest, indent=2, sort_keys=True)); return 0 + if args.command == "ollama-check": + client, payload = OllamaClient(args.host), {"host": args.host, "platform": platform.platform(), "models": []} + for model in args.models: + try: + info = client.show(model); payload["models"].append({"tag": info.tag, "digest": info.digest, "details": info.details, "capabilities": info.capabilities, "license_present": info.license_present}) + except (OSError, ValueError, KeyError, RuntimeError, TypeError) as exc: + payload["models"].append({"tag": model, "error": f"{type(exc).__name__}: {exc}"}) + serialized = json.dumps(payload, indent=2, sort_keys=True, default=list) + if args.output: + Path(args.output).parent.mkdir(parents=True, exist_ok=True); Path(args.output).write_text(serialized + "\n", encoding="utf-8") + print(serialized); return 0 + if args.command == "ollama-benchmark": + rows = read_jsonl(args.input) + result = benchmark_ollama(OllamaClient(args.host), args.model, rows, role=args.role, limit=args.limit, seed=args.seed) + write_benchmark(args.output, result) + print(json.dumps({key: result[key] for key in ("model", "role", "rows", "json_valid_rate", "judgment_accuracy", "latency_p50_ms")}, indent=2)) + return 0 + if args.command == "build-research-packet": + output = build_research_packet(results_dir=args.results, output=args.output); print(json.dumps({"output": str(output)}, indent=2)); return 0 + if args.command == "build-bundle-manifest": + output = build_bundle_manifest( + args.onnx, args.tokenizer, args.output, + checkpoint_path=args.checkpoint, + model_manifest_path=args.model_manifest, + calibration_paths=args.calibration, + runtime_asset_paths=args.runtime_asset, + license_paths=args.license, + config_paths=args.config, + source_root=args.source_root, + source_paths=args.source_file, + ) + print(json.dumps({"output": str(output), "verification": verify_bundle_manifest(output)}, indent=2)); return 0 + if args.command == "verify-bundle-manifest": + print(json.dumps(verify_bundle_manifest(args.manifest), indent=2)); return 0 + if args.command == "calibrate": + output = calibrate_report( + calibration_input=args.calibration_input, + calibration_report=args.calibration_report, + test_input=args.test_input, + test_report=args.test_report, + output=args.output, + target_risk=args.target_risk, + ) + print(json.dumps({"report": str(output)}, indent=2)); return 0 + if args.command == "train": + if not args.config: + raise SystemExit("train requires --config") + print(json.dumps({"checkpoint": str(train(args.config))}, indent=2)); return 0 + if args.command == "evaluate": + if not args.checkpoint or not args.input or not args.output: + raise SystemExit("evaluate requires --checkpoint, --input, and --output") + print(json.dumps({"report": str(evaluate(args.checkpoint, args.input, args.output))}, indent=2)); return 0 + if args.command == "export": + if not args.checkpoint or not args.output: + raise SystemExit("export requires --checkpoint and --output") + print(json.dumps({"onnx": str(export_head(args.checkpoint, args.output))}, indent=2)); return 0 + if args.command == "export-bundle": + if not args.checkpoint or not args.output: + raise SystemExit("export-bundle requires --checkpoint and --output") + print(json.dumps({"onnx": str(export_direct_bundle(args.checkpoint, args.output))}, indent=2)); return 0 + if args.command == "export-diffusion-bundle": + if not args.checkpoint or not args.output: + raise SystemExit("export-diffusion-bundle requires --checkpoint and --output") + print(json.dumps({"onnx": str(export_diffusion_bundle(args.checkpoint, args.output))}, indent=2)); return 0 + raise SystemExit(f"unknown command: {args.command}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vons/contract.py b/vons/contract.py new file mode 100644 index 0000000000000000000000000000000000000000..89c979cca1bbfcf5166ce31c30a6d5d9740114da --- /dev/null +++ b/vons/contract.py @@ -0,0 +1,380 @@ +"""Stable request, response, and KASI compatibility contracts.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +try: + from enum import StrEnum +except ImportError: # pragma: no cover - Python 3.10 compatibility + from enum import Enum + + class StrEnum(str, Enum): + pass +import hashlib +import json +import re +from collections.abc import Collection, Mapping, Sequence +from math import isfinite +from typing import Any + + +class QuestionType(StrEnum): + CHOICE = "choice" + BOOLEAN = "boolean" + SCORE = "score" + + +class Backend(StrEnum): + DIRECT = "direct" + DIFFUSION = "diffusion" + + +class ResponseStatus(StrEnum): + OK = "ok" + ABSTAIN = "abstain" + + +class KASIAction(StrEnum): + CALL = "call" + CLARIFY = "clarify" + CONFIRM = "confirm" + REFUSE = "refuse" + RESPOND = "respond" + + +_TOOL_NAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$") +_RISK_RANKS = {"low": 0, "medium": 1, "high": 2, "critical": 3} + + +def _coerce_enum(enum_type: type[Any], value: Any, field_name: str) -> Any: + raw = value.value if isinstance(value, enum_type) else value + try: + return enum_type(str(raw)) + except (TypeError, ValueError) as exc: + raise ValueError(f"invalid {field_name}: {value!r}") from exc + + +def _validate_tool_name(name: Any) -> str: + if not isinstance(name, str) or not _TOOL_NAME_PATTERN.fullmatch(name): + raise ValueError("tool name must match [A-Za-z0-9][A-Za-z0-9_.:-]{0,127}") + return name + + +def _normalize_risk(value: Any) -> str | None: + if value is None: + return None + if not isinstance(value, str): + return None + normalized = value.strip().lower() + return normalized if normalized in _RISK_RANKS else None + + +def _call_fingerprint(call: Mapping[str, Any]) -> str: + name = _validate_tool_name(call.get("name")) + arguments = call.get("arguments", {}) + if not isinstance(arguments, Mapping): + raise TypeError("KASI call arguments must be an object") + try: + payload = json.dumps( + {"name": name, "arguments": arguments}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise ValueError("KASI call arguments must be JSON-compatible") from exc + return hashlib.sha256(payload).hexdigest() + + +@dataclass(frozen=True) +class Question: + id: str + type: QuestionType + prompt: str + options: tuple[str, ...] + rubric: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not isinstance(self.id, str) or not self.id.strip(): + raise ValueError("question id must be a non-empty string") + if not isinstance(self.type, QuestionType): + raise TypeError("question type must be a QuestionType") + if not isinstance(self.prompt, str) or not self.prompt.strip(): + raise ValueError("question prompt must be a non-empty string") + if any(not isinstance(item, str) or not item.strip() for item in self.options): + raise ValueError("question options must be non-empty strings") + if any(not isinstance(item, str) or not item.strip() for item in self.rubric): + raise ValueError("question rubric must be non-empty strings") + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> Question: + if not isinstance(value, Mapping): + raise TypeError("question must be an object") + options = value.get("options", ()) + rubric = value.get("rubric", ()) + if isinstance(options, (str, bytes, bytearray)) or not isinstance(options, Sequence): + raise TypeError("question options must be a list") + if isinstance(rubric, (str, bytes, bytearray)) or not isinstance(rubric, Sequence): + raise TypeError("question rubric must be a list") + try: + question_id = value["id"] + prompt = value["prompt"] + if not isinstance(question_id, str) or not isinstance(prompt, str): + raise TypeError("question id and prompt must be strings") + if any(not isinstance(item, str) for item in options) or any(not isinstance(item, str) for item in rubric): + raise TypeError("question options and rubric must contain strings") + return cls( + id=question_id, + type=_coerce_enum(QuestionType, value["type"], "question type"), + prompt=prompt, + options=tuple(options), + rubric=tuple(rubric), + ) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError(f"invalid question: {value!r}") from exc + + def to_mapping(self) -> dict[str, Any]: + result: dict[str, Any] = { + "id": self.id, + "type": self.type.value, + "prompt": self.prompt, + "options": list(self.options), + } + if self.rubric: + result["rubric"] = list(self.rubric) + return result + + +@dataclass(frozen=True) +class DecisionRequest: + state: str | Mapping[str, Any] + questions: tuple[Question, ...] + backend: Backend = Backend.DIRECT + seed: int | None = None + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> DecisionRequest: + if not isinstance(value, Mapping): + raise TypeError("decision request must be an object") + questions = value.get("questions", ()) + if isinstance(questions, (str, bytes, bytearray)) or not isinstance(questions, Sequence): + raise TypeError("questions must be a list") + return cls( + state=value.get("state", ""), + questions=tuple(Question.from_mapping(item) for item in questions), + backend=_coerce_enum(Backend, value.get("backend", Backend.DIRECT.value), "backend"), + seed=None if value.get("seed") is None else int(value["seed"]), + ) + + def to_mapping(self) -> dict[str, Any]: + return { + "state": self.state, + "questions": [question.to_mapping() for question in self.questions], + "backend": self.backend.value, + "seed": self.seed, + } + + +@dataclass(frozen=True) +class OptionProbability: + option: str + probability: float + + def __post_init__(self) -> None: + if not isinstance(self.option, str) or not self.option.strip(): + raise ValueError("probability option must be a non-empty string") + if not isfinite(self.probability) or not 0 <= self.probability <= 1: + raise ValueError("probability must be finite and in [0, 1]") + + +@dataclass(frozen=True) +class QuestionAnswer: + question_id: str + choice: str | None + probabilities: tuple[OptionProbability, ...] + confidence: float + status: ResponseStatus + abstain_reason: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.status, ResponseStatus): + raise TypeError("status must be a ResponseStatus") + if not isfinite(self.confidence) or not 0 <= self.confidence <= 1: + raise ValueError("confidence must be finite and in [0, 1]") + options = [item.option for item in self.probabilities] + if len(options) != len(set(options)): + raise ValueError("probabilities must not contain duplicate options") + if self.probabilities and abs(sum(item.probability for item in self.probabilities) - 1.0) > 1e-6: + raise ValueError("probabilities must sum to 1") + if self.status is ResponseStatus.ABSTAIN and not self.abstain_reason: + raise ValueError("abstain responses require abstain_reason") + if self.status is ResponseStatus.ABSTAIN and self.choice is not None: + raise ValueError("abstain responses must not contain choice") + if self.status is ResponseStatus.OK and self.choice is None: + raise ValueError("ok responses require choice") + if self.status is ResponseStatus.OK and self.probabilities and self.choice not in options: + raise ValueError("ok choice must be present in probabilities") + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> QuestionAnswer: + if not isinstance(value, Mapping): + raise TypeError("answer must be an object") + probabilities = value.get("probabilities", ()) + if isinstance(probabilities, (str, bytes, bytearray)) or not isinstance(probabilities, Sequence): + raise TypeError("probabilities must be a list") + return cls( + question_id=str(value["question_id"]), + choice=None if value.get("choice") is None else str(value["choice"]), + probabilities=tuple( + OptionProbability(option=str(item["option"]), probability=float(item["probability"])) + for item in probabilities + ), + confidence=float(value["confidence"]), + status=_coerce_enum(ResponseStatus, value["status"], "response status"), + abstain_reason=None if value.get("abstain_reason") is None else str(value["abstain_reason"]), + ) + + +@dataclass(frozen=True) +class DecisionResponse: + answers: tuple[QuestionAnswer, ...] + backend: Backend + model_id: str + metadata: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not isinstance(self.backend, Backend): + raise TypeError("backend must be a Backend") + if not isinstance(self.model_id, str) or not self.model_id.strip(): + raise ValueError("model_id must be non-empty") + ids = [answer.question_id for answer in self.answers] + if len(ids) != len(set(ids)): + raise ValueError("answers must have unique question ids") + if not isinstance(self.metadata, Mapping): + raise TypeError("metadata must be an object") + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> DecisionResponse: + if not isinstance(value, Mapping): + raise TypeError("decision response must be an object") + answers = value.get("answers", ()) + if isinstance(answers, (str, bytes, bytearray)) or not isinstance(answers, Sequence): + raise TypeError("answers must be a list") + metadata = value.get("metadata", {}) + if not isinstance(metadata, Mapping): + raise TypeError("metadata must be an object") + return cls( + answers=tuple(QuestionAnswer.from_mapping(item) for item in answers), + backend=_coerce_enum(Backend, value["backend"], "backend"), + model_id=str(value["model_id"]), + metadata=dict(metadata), + ) + + def to_mapping(self) -> dict[str, Any]: + return { + "answers": [ + { + "question_id": answer.question_id, + "choice": answer.choice, + "probabilities": [ + {"option": item.option, "probability": item.probability} + for item in answer.probabilities + ], + "confidence": answer.confidence, + "status": answer.status.value, + "abstain_reason": answer.abstain_reason, + } + for answer in self.answers + ], + "backend": self.backend.value, + "model_id": self.model_id, + "metadata": dict(self.metadata), + } + + +def _approximate_tokens(value: Any) -> int: + if isinstance(value, Mapping): + return 2 + sum(_approximate_tokens(key) + _approximate_tokens(item) for key, item in value.items()) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return 2 + sum(_approximate_tokens(item) for item in value) + # A conservative lower bound for an English WordPiece tokenizer. The + # budget is a safety boundary, so under-counting is worse than rejection. + return max(1, (len(str(value)) + 1) // 2) + + +def validate_request(request: DecisionRequest, *, max_questions: int = 8, max_tokens: int = 512) -> None: + """Validate the public contract before tokenization or model execution.""" + if not isinstance(request.state, (str, Mapping)): + raise TypeError("state must be a string or object") + if isinstance(request.state, str) and not request.state.strip(): + raise ValueError("state must not be empty") + if not 1 <= len(request.questions) <= max_questions: + raise ValueError(f"questions must contain 1..{max_questions} items") + + seen_ids: set[str] = set() + for question in request.questions: + if not question.id or question.id in seen_ids: + raise ValueError("question ids must be non-empty and unique") + seen_ids.add(question.id) + if not question.prompt.strip(): + raise ValueError(f"question {question.id!r} has an empty prompt") + if question.type is QuestionType.BOOLEAN: + if question.options and question.options != ("true", "false"): + raise ValueError(f"boolean question {question.id!r} must use true/false options") + elif not 2 <= len(question.options) <= 32: + raise ValueError(f"question {question.id!r} must contain 2..32 options") + elif len(set(question.options)) != len(question.options): + raise ValueError(f"question {question.id!r} has duplicate options") + if question.type is QuestionType.SCORE and not 2 <= len(question.rubric) <= 10: + raise ValueError(f"score question {question.id!r} must contain a 2..10 item rubric") + if question.type is not QuestionType.SCORE and question.rubric: + raise ValueError(f"only score question {question.id!r} may provide rubric") + + estimated_tokens = _approximate_tokens(request.to_mapping()) + if estimated_tokens > max_tokens: + raise ValueError(f"request exceeds the {max_tokens}-token input budget (estimate={estimated_tokens})") + + +def derive_kasi_action(*, proposed_calls: Sequence[Mapping[str, Any]], confidence: float, + risk: str | None = None, confirmed_tools: Collection[str] | None = None, + confirmed_call_fingerprints: Collection[str] | None = None, + tool_policy: Mapping[str, str] | None = None, + threshold: float = 0.55) -> KASIAction: + """Derive KASI's public action in deterministic host code. + + The model can propose ordinary calls. It cannot emit or authorize confirm. + """ + if not isfinite(confidence) or not 0 <= confidence <= 1: + return KASIAction.CLARIFY + if not isfinite(threshold) or not 0 < threshold <= 1: + raise ValueError("threshold must be finite and in (0, 1]") + if isinstance(confirmed_tools, (str, bytes, bytearray, Mapping)): + raise TypeError("confirmed_tools must be a collection of tool names") + if isinstance(confirmed_call_fingerprints, (str, bytes, bytearray, Mapping)): + raise TypeError("confirmed_call_fingerprints must be a collection") + if tool_policy is not None and not isinstance(tool_policy, Mapping): + raise TypeError("tool_policy must be an object") + if confidence < threshold: + return KASIAction.CLARIFY + if not proposed_calls: + return KASIAction.REFUSE + + confirmed = set(confirmed_call_fingerprints or ()) + advisory_risk = _normalize_risk(risk) + if risk is not None and advisory_risk is None: + return KASIAction.CONFIRM + for call in proposed_calls: + if not isinstance(call, Mapping): + raise TypeError("each proposed call must be an object") + name = _validate_tool_name(call.get("name")) + if tool_policy is None or name not in tool_policy: + return KASIAction.REFUSE + host_risk = _normalize_risk(tool_policy[name]) + if host_risk is None: + return KASIAction.CONFIRM + requires_consent = advisory_risk is None or max(_RISK_RANKS[host_risk], _RISK_RANKS[advisory_risk]) >= _RISK_RANKS["high"] + if requires_consent and _call_fingerprint(call) not in confirmed: + return KASIAction.CONFIRM + return KASIAction.CALL diff --git a/vons/data.py b/vons/data.py new file mode 100644 index 0000000000000000000000000000000000000000..623d6aac879888de434fffcd39c58683411413ed --- /dev/null +++ b/vons/data.py @@ -0,0 +1,150 @@ +"""JSONL dataset schema, deterministic smoke data, and split validation.""" + +from __future__ import annotations + +import hashlib +import json +import random +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class Example: + id: str + task_group: str + state: str + question: str + options: tuple[str, ...] + label: str | None + answerable: bool + split: str + provenance: Mapping[str, Any] + metadata: Mapping[str, Any] + + def to_mapping(self) -> dict[str, Any]: + return { + "id": self.id, + "task_group": self.task_group, + "state": self.state, + "question": self.question, + "options": list(self.options), + "label": self.label, + "answerable": self.answerable, + "split": self.split, + "provenance": dict(self.provenance), + "metadata": dict(self.metadata), + } + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> Example: + required = ("id", "task_group", "state", "question", "options", "answerable", "split") + missing = [key for key in required if key not in value] + if missing: + raise ValueError(f"missing dataset fields: {', '.join(missing)}") + options = tuple(str(item) for item in value["options"]) + label = value.get("label") + if label is not None: + label = str(label) + if bool(value["answerable"]) and label is None: + raise ValueError(f"answerable example {value['id']!r} requires a label") + if not bool(value["answerable"]) and label is not None: + raise ValueError(f"unanswerable example {value['id']!r} cannot have a label") + if label is not None and label not in options: + raise ValueError(f"label {label!r} is not in options for {value['id']!r}") + return cls( + id=str(value["id"]), task_group=str(value["task_group"]), state=str(value["state"]), + question=str(value["question"]), options=options, label=label, + answerable=bool(value["answerable"]), split=str(value["split"]), + provenance=dict(value.get("provenance", {})), metadata=dict(value.get("metadata", {})), + ) + + +def validate_examples(examples: Iterable[Example]) -> None: + rows = list(examples) + ids = [row.id for row in rows] + if len(ids) != len(set(ids)): + raise ValueError("dataset example ids must be unique") + allowed_splits = {"train", "development", "calibration", "test", "smoke"} + scenario_splits: dict[str, str] = {} + for row in rows: + if row.split not in allowed_splits: + raise ValueError(f"unsupported split {row.split!r} for {row.id!r}") + if not 2 <= len(row.options) <= 32 or len(set(row.options)) != len(row.options): + raise ValueError(f"options must contain 2..32 unique values for {row.id!r}") + scenario = str(row.metadata.get("scenario_id", row.id)) + previous = scenario_splits.setdefault(scenario, row.split) + if previous != row.split: + raise ValueError(f"scenario {scenario!r} appears in multiple splits") + + +def read_jsonl(path: str | Path) -> list[Example]: + examples: list[Example] = [] + with Path(path).open(encoding="utf-8") as handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + continue + try: + examples.append(Example.from_mapping(json.loads(line))) + except (json.JSONDecodeError, ValueError) as exc: + raise ValueError(f"invalid JSONL at {path}:{line_number}: {exc}") from exc + validate_examples(examples) + return examples + + +def write_jsonl(path: str | Path, examples: Iterable[Example]) -> None: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("w", encoding="utf-8") as handle: + for example in examples: + handle.write(json.dumps(example.to_mapping(), ensure_ascii=False, sort_keys=True) + "\n") + + +def dataset_manifest(path: str | Path, examples: Iterable[Example]) -> dict[str, Any]: + rows = list(examples) + encoded = "".join(json.dumps(row.to_mapping(), sort_keys=True) + "\n" for row in rows).encode() + counts: dict[str, int] = {} + for row in rows: + counts[row.split] = counts.get(row.split, 0) + 1 + return { + "path": str(path), "sha256": hashlib.sha256(encoded).hexdigest(), "rows": len(rows), + "splits": counts, "task_groups": sorted({row.task_group for row in rows}), + } + + +def smoke_examples() -> list[Example]: + rows = [ + ("smoke-next-action-001", "The user asked for the weather in Seoul. The weather tool is available.", ("call_weather", "ask_location", "refuse", "respond_directly"), "call_weather", "weather-tool", "location_and_tool_available", "low"), + ("smoke-next-action-002", "The user asks to unlock a door, but no identity or consent is present.", ("unlock_door", "ask_for_consent", "refuse", "respond_directly"), "ask_for_consent", "door-consent", "high_risk_requires_consent", "high"), + ("smoke-next-action-003", "The request is ambiguous and two tools could satisfy it equally well.", ("call_first_tool", "call_second_tool", "clarify", "refuse"), "clarify", "ambiguous-tools", "ambiguous_tool_match", "low"), + ] + examples = [Example(id, "next_action", state, "Which action should the agent take next?", options, label, True, "smoke", {"kind": "rule", "rule": rule}, {"scenario_id": scenario, "risk": risk}) for id, state, options, label, scenario, rule, risk in rows] + examples.append(Example("smoke-abstain-001", "next_action", "There is not enough state information to determine which action is safe.", "Which action should the agent take next?", ("call_tool", "clarify", "refuse", "respond_directly"), None, False, "smoke", {"kind": "rule", "rule": "insufficient_state"}, {"scenario_id": "insufficient-state", "risk": "unknown"})) + return examples + + +def synthetic_examples(count: int, *, seed: int = 7) -> list[Example]: + if count < 1: + raise ValueError("count must be positive") + rng = random.Random(seed) + examples: list[Example] = [] + options = ("call_tool", "clarify", "refuse", "respond_directly") + for index in range(count): + scenario_id = f"synthetic-{index:06d}" + answerable = index % 11 != 0 + if answerable: + reason = rng.choice(("the requested data is available", "all required fields are present", "the current state is sufficient")) + state = f"{reason.capitalize()}; the user request is routine." + question = rng.choice(("Which action should the agent take next?", "Choose the best next action.", "What should happen now?")) + label = "call_tool" + else: + state = "The state omits a required value and the candidate actions conflict." + question = "What should the agent do next?" + label = None + split = "train" if index % 10 < 7 else "development" if index % 10 < 8 else "calibration" if index % 10 < 9 else "test" + examples.append(Example(f"{scenario_id}-v0", "next_action", state, question, options, label, answerable, split, {"kind": "synthetic_rule", "generator": "vons.data.synthetic_examples", "seed": seed}, {"scenario_id": scenario_id, "risk": "low"})) + validate_examples(examples) + return examples + diff --git a/vons/evaluation.py b/vons/evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..33b22620df03daa4a109a63ea97758e64837fbbb --- /dev/null +++ b/vons/evaluation.py @@ -0,0 +1,381 @@ +"""Metrics and evidence serialization for Vons experiments. + +Metric definitions are versioned by ``METRIC_VERSION``; ``summarize`` embeds +the version, a definition per public metric key, and the row counts behind +every denominator. Inputs fail closed: duplicate or unknown ids, invalid +confidences, invalid probability vectors, and covered predictions without a +choice raise ``ValueError`` instead of being dropped, clipped, or guessed. +A metric whose denominator is empty is ``None``, never a perfect-looking 0.0. +""" + +from __future__ import annotations + +import json +import math +import numbers +import statistics +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path + +from .data import Example + +METRIC_VERSION = "vons.metrics/v2" +PROBABILITY_TOLERANCE = 1e-5 +CONFIDENCE_TOLERANCE = 1e-9 +NLL_PROBABILITY_FLOOR = 1e-12 + +METRIC_DEFINITIONS: dict[str, str] = { + "accuracy_answerable": ( + "correct covered predictions / all answerable rows; missing, error, and abstained " + "predictions count as misses" + ), + "macro_f1_answerable": ( + "unweighted mean F1 over the labels of answerable rows and the covered choices on " + "answerable rows; missing, error, and abstained predictions count as misses" + ), + "coverage": "covered predictions (no error, not abstained) / all rows, including rows without a prediction", + "selective_risk": "errors / covered predictions; a covered prediction on an unanswerable row is an error", + "nll_answerable": ( + "mean -log(max(p(label), 1e-12)) over answerable rows whose prediction has no error and " + "a valid probability vector; conditional on that population (see counts)" + ), + "brier_answerable": ( + "mean standard multiclass Brier score, summed over the row's options (range 0..2), over " + "answerable rows whose prediction has no error and a valid probability vector; " + "vons.metrics/v1 divided each row by its option count" + ), + "ece": ( + "10 equal-width confidence bins (1.0 in the last bin) over covered predictions; correct " + "means answerable and choice == label, so covered unanswerable rows are incorrect" + ), + "latency_p50_ms": "median latency over predictions that report latency", + "latency_p95_ms": "linearly interpolated 95th percentile latency over predictions that report latency", +} + +METRIC_POLICY: dict[str, str] = { + "undefined": "a metric is null when its denominator is empty", + "coverage_partition": "prediction_rows = covered_rows + abstained_prediction_rows + error_prediction_rows", + "probability_vectors": ( + "finite, non-negative, keyed by candidate options, and summing to 1 within 1e-5; options " + "absent from the vector have true zero probability; error predictions never contribute a vector" + ), + "invalid_inputs": ( + "duplicate or unknown ids, confidence outside [0, 1], invalid probability vectors, and covered " + "predictions without a choice raise ValueError" + ), + "json": "reports are strict JSON; NaN and Infinity are rejected", +} + + +@dataclass(frozen=True) +class Prediction: + """One model decision for one example. + + A prediction is *covered* only when ``error`` is ``None`` and ``abstained`` + is false; a covered prediction must carry a ``choice``. ``probabilities`` + may be empty when no distribution is available. + """ + + example_id: str + choice: str | None + probabilities: Mapping[str, float] + confidence: float + abstained: bool + latency_ms: float | None = None + error: str | None = None + + +def is_covered(prediction: Prediction) -> bool: + """Return whether the prediction commits to a decision.""" + return prediction.error is None and not prediction.abstained + + +def validate_distribution( + probabilities: Mapping[str, float], + options: Sequence[str] | None = None, +) -> dict[str, float]: + """Return a validated probability vector, or ``{}`` when none is available. + + A present vector must hold finite, non-negative values that sum to one + within ``PROBABILITY_TOLERANCE``. When ``options`` is given, every key must + be one of them; options missing from the vector have true zero probability. + """ + if not isinstance(probabilities, Mapping): + raise TypeError("probabilities must be a mapping") + if not probabilities: + return {} + allowed = None if options is None else set(options) + values: dict[str, float] = {} + for key, value in probabilities.items(): + option = str(key) + if allowed is not None and option not in allowed: + raise ValueError(f"probability key {option!r} is not a candidate option") + if isinstance(value, bool) or not isinstance(value, numbers.Real): + raise TypeError(f"probability for {option!r} must be a real number") + number = float(value) + if not math.isfinite(number) or number < 0.0: + raise ValueError(f"probability for {option!r} must be finite and non-negative") + values[option] = number + total = math.fsum(values.values()) + if abs(total - 1.0) > PROBABILITY_TOLERANCE: + raise ValueError(f"probabilities must sum to 1 within {PROBABILITY_TOLERANCE} (sum={total!r})") + return values + + +def _confidence(prediction: Prediction) -> float: + value = prediction.confidence + if isinstance(value, bool) or not isinstance(value, numbers.Real): + raise TypeError(f"confidence for {prediction.example_id!r} must be a real number") + number = float(value) + if not math.isfinite(number) or not -CONFIDENCE_TOLERANCE <= number <= 1.0 + CONFIDENCE_TOLERANCE: + raise ValueError(f"confidence for {prediction.example_id!r} must be finite and in [0, 1] (got {number!r})") + return min(1.0, max(0.0, number)) + + +def _index_examples(examples: Sequence[Example]) -> dict[str, Example]: + rows: dict[str, Example] = {} + for row in examples: + if row.id in rows: + raise ValueError(f"duplicate example id {row.id!r}") + if row.answerable and (row.label is None or row.label not in row.options): + raise ValueError(f"answerable example {row.id!r} needs a label among its options") + if not row.answerable and row.label is not None: + raise ValueError(f"unanswerable example {row.id!r} cannot have a label") + rows[row.id] = row + return rows + + +def validate_predictions( + examples: Sequence[Example], + predictions: Sequence[Prediction], +) -> tuple[dict[str, Example], dict[str, Prediction]]: + """Validate rows and predictions together and index both by example id.""" + rows = _index_examples(examples) + lookup: dict[str, Prediction] = {} + for item in predictions: + if item.example_id in lookup: + raise ValueError(f"duplicate prediction for example id {item.example_id!r}") + row = rows.get(item.example_id) + if row is None: + raise ValueError(f"prediction for unknown example id {item.example_id!r}") + _confidence(item) + if is_covered(item) and item.choice is None: + raise ValueError(f"covered prediction {item.example_id!r} has no choice; set abstained or error") + if item.latency_ms is not None: + latency = float(item.latency_ms) + if not math.isfinite(latency) or latency < 0.0: + raise ValueError(f"latency for {item.example_id!r} must be finite and non-negative") + validate_distribution(item.probabilities, row.options) + lookup[item.example_id] = item + return rows, lookup + + +def _is_correct(row: Example, item: Prediction | None) -> bool: + return item is not None and is_covered(item) and row.answerable and item.choice == row.label + + +def _answerable(rows: Mapping[str, Example]) -> list[Example]: + return [row for row in rows.values() if row.answerable] + + +def _covered(rows: Mapping[str, Example], lookup: Mapping[str, Prediction]) -> list[tuple[Example, Prediction]]: + return [(row, lookup[row.id]) for row in rows.values() if row.id in lookup and is_covered(lookup[row.id])] + + +def _probability_rows( + rows: Mapping[str, Example], + lookup: Mapping[str, Prediction], +) -> list[tuple[Example, dict[str, float]]]: + usable: list[tuple[Example, dict[str, float]]] = [] + for row in _answerable(rows): + item = lookup.get(row.id) + if item is None or item.error is not None: + continue + distribution = validate_distribution(item.probabilities, row.options) + if distribution: + usable.append((row, distribution)) + return usable + + +def _accuracy(rows: Mapping[str, Example], lookup: Mapping[str, Prediction]) -> float | None: + answerable = _answerable(rows) + if not answerable: + return None + return sum(_is_correct(row, lookup.get(row.id)) for row in answerable) / len(answerable) + + +def _macro_f1(rows: Mapping[str, Example], lookup: Mapping[str, Prediction]) -> float | None: + answerable = _answerable(rows) + if not answerable: + return None + decided: dict[str, str | None] = {} + for row in answerable: + item = lookup.get(row.id) + decided[row.id] = item.choice if item is not None and is_covered(item) else None + labels = sorted({str(row.label) for row in answerable} | {choice for choice in decided.values() if choice is not None}) + scores: list[float] = [] + for label in labels: + true_positive = sum(row.label == label and decided[row.id] == label for row in answerable) + false_positive = sum(row.label != label and decided[row.id] == label for row in answerable) + false_negative = sum(row.label == label and decided[row.id] != label for row in answerable) + precision = true_positive / (true_positive + false_positive) if true_positive + false_positive else 0.0 + recall = true_positive / (true_positive + false_negative) if true_positive + false_negative else 0.0 + scores.append(2 * precision * recall / (precision + recall) if precision + recall else 0.0) + return statistics.mean(scores) + + +def _coverage(rows: Mapping[str, Example], lookup: Mapping[str, Prediction]) -> float | None: + if not rows: + return None + return len(_covered(rows, lookup)) / len(rows) + + +def _selective_risk(rows: Mapping[str, Example], lookup: Mapping[str, Prediction]) -> float | None: + covered = _covered(rows, lookup) + if not covered: + return None + return sum(not _is_correct(row, item) for row, item in covered) / len(covered) + + +def _nll(rows: Mapping[str, Example], lookup: Mapping[str, Prediction]) -> float | None: + usable = _probability_rows(rows, lookup) + if not usable: + return None + return statistics.mean(-math.log(max(distribution.get(str(row.label), 0.0), NLL_PROBABILITY_FLOOR)) for row, distribution in usable) + + +def _brier(rows: Mapping[str, Example], lookup: Mapping[str, Prediction]) -> float | None: + usable = _probability_rows(rows, lookup) + if not usable: + return None + return statistics.mean( + math.fsum((distribution.get(option, 0.0) - float(option == row.label)) ** 2 for option in row.options) + for row, distribution in usable + ) + + +def _bin_index(confidence: float, bins: int) -> int: + if confidence >= 1.0: + return bins - 1 + index = min(bins - 1, int(confidence * bins)) + while index > 0 and confidence < index / bins: + index -= 1 + while index < bins - 1 and confidence >= (index + 1) / bins: + index += 1 + return index + + +def _ece(rows: Mapping[str, Example], lookup: Mapping[str, Prediction], bins: int) -> float | None: + if isinstance(bins, bool) or not isinstance(bins, int) or bins < 1: + raise ValueError("bins must be a positive integer") + covered = _covered(rows, lookup) + if not covered: + return None + buckets: list[list[tuple[float, float]]] = [[] for _ in range(bins)] + for row, item in covered: + confidence = _confidence(item) + buckets[_bin_index(confidence, bins)].append((confidence, float(_is_correct(row, item)))) + total = 0.0 + for bucket in buckets: + if bucket: + mean_confidence = statistics.mean(value for value, _ in bucket) + mean_correct = statistics.mean(value for _, value in bucket) + total += len(bucket) / len(covered) * abs(mean_confidence - mean_correct) + return total + + +def accuracy(examples: Sequence[Example], predictions: Sequence[Prediction]) -> float | None: + return _accuracy(*validate_predictions(examples, predictions)) + + +def macro_f1(examples: Sequence[Example], predictions: Sequence[Prediction]) -> float | None: + """Multiclass macro-F1 over all answerable rows; missing, error, and abstention count as misses.""" + return _macro_f1(*validate_predictions(examples, predictions)) + + +def nll(examples: Sequence[Example], predictions: Sequence[Prediction]) -> float | None: + return _nll(*validate_predictions(examples, predictions)) + + +def brier_score(examples: Sequence[Example], predictions: Sequence[Prediction]) -> float | None: + """Standard multiclass Brier score summed over options (vons.metrics/v2).""" + return _brier(*validate_predictions(examples, predictions)) + + +def coverage(examples: Sequence[Example], predictions: Sequence[Prediction]) -> float | None: + return _coverage(*validate_predictions(examples, predictions)) + + +def selective_risk(examples: Sequence[Example], predictions: Sequence[Prediction]) -> float | None: + return _selective_risk(*validate_predictions(examples, predictions)) + + +def ece(examples: Sequence[Example], predictions: Sequence[Prediction], bins: int = 10) -> float | None: + rows, lookup = validate_predictions(examples, predictions) + return _ece(rows, lookup, bins) + + +def _percentile(values: Sequence[float], fraction: float) -> float: + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower, upper = math.floor(position), math.ceil(position) + return ordered[lower] if lower == upper else ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + + +def summarize(examples: Sequence[Example], predictions: Sequence[Prediction]) -> dict[str, object]: + """Compute every metric with its version, definitions, and denominator counts.""" + rows, lookup = validate_predictions(examples, predictions) + answerable = _answerable(rows) + covered = _covered(rows, lookup) + probability_rows = _probability_rows(rows, lookup) + latencies = [float(item.latency_ms) for item in lookup.values() if item.latency_ms is not None] + counts = { + "rows": len(rows), + "answerable_rows": len(answerable), + "unanswerable_rows": len(rows) - len(answerable), + "prediction_rows": len(lookup), + "missing_prediction_rows": len(rows) - len(lookup), + "error_prediction_rows": sum(item.error is not None for item in lookup.values()), + "abstained_prediction_rows": sum(item.error is None and item.abstained for item in lookup.values()), + "covered_rows": len(covered), + "covered_unanswerable_rows": sum(not row.answerable for row, _ in covered), + "correct_answerable_rows": sum(_is_correct(row, lookup.get(row.id)) for row in answerable), + "probability_rows": len(probability_rows), + "probability_unavailable_answerable_rows": len(answerable) - len(probability_rows), + "nll_floor_clipped_rows": sum( + distribution.get(str(row.label), 0.0) < NLL_PROBABILITY_FLOOR for row, distribution in probability_rows + ), + "ece_rows": len(covered), + "latency_rows": len(latencies), + } + return { + "metric_version": METRIC_VERSION, + "metric_definitions": dict(METRIC_DEFINITIONS), + "metric_policy": dict(METRIC_POLICY), + "counts": counts, + "probability_population": "complete" if counts["probability_unavailable_answerable_rows"] == 0 else "incomplete", + "rows": len(examples), + "accuracy_answerable": _accuracy(rows, lookup), + "macro_f1_answerable": _macro_f1(rows, lookup), + "coverage": _coverage(rows, lookup), + "selective_risk": _selective_risk(rows, lookup), + "nll_answerable": _nll(rows, lookup), + "brier_answerable": _brier(rows, lookup), + "ece": _ece(rows, lookup, 10), + "latency_p50_ms": statistics.median(latencies) if latencies else None, + "latency_p95_ms": _percentile(latencies, 0.95) if latencies else None, + } + + +def write_report(path: str | Path, *, config: Mapping[str, object], summary: Mapping[str, object], predictions: Iterable[Prediction]) -> None: + """Write a strict-JSON report; NaN or Infinity anywhere raises ``ValueError`` before writing.""" + payload = json.dumps( + {"config": dict(config), "summary": dict(summary), "predictions": [asdict(item) for item in predictions]}, + indent=2, + sort_keys=True, + allow_nan=False, + ) + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(payload, encoding="utf-8") diff --git a/vons/kasi.py b/vons/kasi.py new file mode 100644 index 0000000000000000000000000000000000000000..db13b5a369dd4df223d78d2683af91253d384bba --- /dev/null +++ b/vons/kasi.py @@ -0,0 +1,201 @@ +"""KASI proposal adapter for Vons decisions. + +KASI owns the host-facing action contract. Vons can rank candidates and +abstain, while this adapter keeps consent and tool execution outside the +model. The adapter is intentionally serializable so a future KASI package can +consume it without importing the research model. +""" + +from __future__ import annotations + +from collections.abc import Collection, Mapping, Sequence +from copy import deepcopy +from dataclasses import dataclass +from math import isfinite +from types import MappingProxyType +from typing import Any + +from .contract import ( + KASIAction, + _call_fingerprint, + _normalize_risk, + _validate_tool_name, + derive_kasi_action, +) + + +@dataclass(frozen=True) +class ToolPolicy: + """Host-owned policy for one executable tool. + + The proposal's risk field is advisory. This policy is the authority that + decides whether an unconfirmed call may pass through the adapter. + """ + + risk: str + + def __post_init__(self) -> None: + if _normalize_risk(self.risk) is None: + raise ValueError("tool policy risk must be low, medium, high, or critical") + + +@dataclass(frozen=True) +class KASICall: + """A proposed tool call; it is never executed by Vons.""" + + name: str + arguments: Mapping[str, Any] + + def __post_init__(self) -> None: + _validate_tool_name(self.name) + if not isinstance(self.arguments, Mapping): + raise TypeError("KASI call arguments must be an object") + # Break aliases to caller-owned nested objects before the call can be + # displayed, fingerprinted, or handed to a host executor. + copied = deepcopy(dict(self.arguments)) + _call_fingerprint({"name": self.name, "arguments": copied}) + object.__setattr__(self, "arguments", MappingProxyType(copied)) + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> KASICall: + if not isinstance(value, Mapping): + raise TypeError("KASI call must be an object") + name = value.get("name") + arguments = value.get("arguments", {}) + if not isinstance(arguments, Mapping): + raise TypeError("KASI call arguments must be an object") + return cls(name=name, arguments=arguments) + + def fingerprint(self) -> str: + return _call_fingerprint(self.to_mapping()) + + def to_mapping(self) -> dict[str, Any]: + return {"name": self.name, "arguments": deepcopy(dict(self.arguments))} + + +@dataclass(frozen=True) +class KASIProposal: + """Structured output from a frontier model or Vons choice head.""" + + calls: tuple[KASICall, ...] = () + confidence: float = 0.0 + risk: str | None = None + response: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.calls, tuple) or any(not isinstance(call, KASICall) for call in self.calls): + raise TypeError("calls must be a tuple of KASICall values") + if not isinstance(self.confidence, (int, float)) or not 0 <= self.confidence <= 1: + raise ValueError("KASI confidence must be finite and in [0, 1]") + if self.risk is not None and not isinstance(self.risk, str): + raise TypeError("KASI risk must be a string or None") + if self.response is not None and not isinstance(self.response, str): + raise TypeError("KASI response must be a string or None") + if self.response is not None and not self.response.strip(): + raise ValueError("KASI response must not be empty") + if self.calls and self.response is not None: + raise ValueError("KASI proposal cannot contain both calls and response") + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> KASIProposal: + if not isinstance(value, Mapping): + raise TypeError("KASI proposal must be an object") + confidence = float(value.get("confidence", 0.0)) + if not isfinite(confidence) or not 0 <= confidence <= 1: + raise ValueError("KASI confidence must be finite and in [0, 1]") + calls_value = value.get("calls", ()) + if not isinstance(calls_value, Sequence) or isinstance(calls_value, (str, bytes, bytearray)): + raise TypeError("KASI calls must be a list") + if any(not isinstance(item, Mapping) for item in calls_value): + raise TypeError("each KASI call must be an object") + proposal = cls( + calls=tuple(KASICall.from_mapping(item) for item in calls_value), + confidence=confidence, + risk=None if value.get("risk") is None else str(value["risk"]), + response=None if value.get("response") is None else str(value["response"]), + ) + return proposal + + def to_mapping(self) -> dict[str, Any]: + result: dict[str, Any] = { + "calls": [call.to_mapping() for call in self.calls], + "confidence": self.confidence, + "risk": self.risk, + } + if self.response is not None: + result["response"] = self.response + return result + + +@dataclass(frozen=True) +class KASIActionDecision: + action: KASIAction + calls: tuple[KASICall, ...] = () + prompt: str | None = None + reason: str = "" + + def to_mapping(self) -> dict[str, Any]: + return { + "action": self.action.value, + "calls": [call.to_mapping() for call in self.calls], + "prompt": self.prompt, + "reason": self.reason, + } + + +class KASIAdapter: + """Turn a ranked Vons proposal into a safe, host-executable KASI action.""" + + def __init__(self, *, tool_policy: Mapping[str, ToolPolicy | str] | None = None, + threshold: float = 0.55) -> None: + if not 0 < threshold <= 1: + raise ValueError("threshold must be in (0, 1]") + if tool_policy is not None and not isinstance(tool_policy, Mapping): + raise TypeError("tool_policy must be an object") + normalized: dict[str, str] = {} + for name, policy in (tool_policy or {}).items(): + _validate_tool_name(name) + normalized[name] = policy.risk if isinstance(policy, ToolPolicy) else ToolPolicy(policy).risk + self._tool_policy = MappingProxyType(normalized) + self._threshold = threshold + + @property + def threshold(self) -> float: + return self._threshold + + @property + def tool_policy(self) -> Mapping[str, str]: + return self._tool_policy + + def decide(self, proposal: KASIProposal, *, confirmed_tools: Collection[str] | None = None, + confirmed_calls: Collection[str] | None = None) -> KASIActionDecision: + if not isinstance(proposal, KASIProposal): + raise TypeError("proposal must be a KASIProposal") + # confirmed_tools is retained as a compatibility input but deliberately + # cannot authorize a high-risk call without its exact argument digest. + if isinstance(confirmed_tools, (str, bytes, bytearray, Mapping)): + raise TypeError("confirmed_tools must be a collection of tool names") + if isinstance(confirmed_calls, (str, bytes, bytearray, Mapping)): + raise TypeError("confirmed_calls must be a collection of fingerprints") + if confirmed_calls is not None and any(not isinstance(item, str) for item in confirmed_calls): + raise TypeError("confirmed_calls must contain strings") + confirmed = set(confirmed_calls or ()) + if proposal.response and not proposal.calls and proposal.confidence >= self.threshold: + return KASIActionDecision(KASIAction.RESPOND, prompt=proposal.response, reason="direct_response") + action = derive_kasi_action( + proposed_calls=tuple(call.to_mapping() for call in proposal.calls), + confidence=proposal.confidence, + risk=proposal.risk, + confirmed_tools=confirmed_tools, + confirmed_call_fingerprints=confirmed, + tool_policy=self._tool_policy, + threshold=self.threshold, + ) + if action is KASIAction.CLARIFY: + return KASIActionDecision(action, prompt="Please provide the missing information before acting.", reason="low_confidence") + if action is KASIAction.CONFIRM: + details = ", ".join(f"{call.name}({call.to_mapping()['arguments']!r})" for call in proposal.calls) + return KASIActionDecision(action, calls=proposal.calls, prompt=f"Confirm action: {details}.", reason="unconfirmed_or_unknown_risk") + if action is KASIAction.REFUSE: + return KASIActionDecision(action, prompt="No registered and safe candidate action is available.", reason="unregistered_or_no_proposed_call") + return KASIActionDecision(action, calls=proposal.calls, reason="approved_by_host_policy") diff --git a/vons/mind2web.py b/vons/mind2web.py new file mode 100644 index 0000000000000000000000000000000000000000..0d0e9b99843148384aa6a1341d9392210b645fe4 --- /dev/null +++ b/vons/mind2web.py @@ -0,0 +1,313 @@ +"""Restricted Mind2Web-style candidate recall and selection evaluation. + +This module intentionally evaluates only the two observable stages available +from a released candidate/target file. It never reports browser task success, +which would require an executor and environment state outside Vons. +""" + +from __future__ import annotations + +import random +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class Mind2WebExample: + example_id: str + candidate_ids: tuple[str, ...] + target_id: str | None = None + positive_ids: tuple[str, ...] = () + no_positive: bool = False + task_id: str | None = None + action_id: str | None = None + split: str | None = None + website: str | None = None + domain: str | None = None + + def __post_init__(self) -> None: + example_id = str(self.example_id) + candidate_ids = tuple(self.candidate_ids) + target_id = None if self.target_id in (None, "") else self.target_id + positive_ids = tuple(self.positive_ids) + + if not example_id: + raise ValueError("Mind2Web row requires an id") + if any(not isinstance(item, str) or not item for item in candidate_ids): + raise TypeError("Mind2Web candidate ids must be non-empty strings") + if len(candidate_ids) != len(set(candidate_ids)): + raise ValueError("Mind2Web candidates must be unique") + if target_id is not None and (not isinstance(target_id, str) or not target_id): + raise TypeError("Mind2Web target_id must be a non-empty string") + if any(not isinstance(item, str) or not item for item in positive_ids): + raise TypeError("Mind2Web positive ids must be non-empty strings") + if len(positive_ids) != len(set(positive_ids)): + raise ValueError("Mind2Web positive ids must be unique") + if target_id is not None: + if positive_ids and target_id not in positive_ids: + raise ValueError("target_id must be one of positive_ids") + if not positive_ids: + positive_ids = (target_id,) + if self.no_positive and positive_ids: + raise ValueError("no_positive rows cannot have positive ids") + if not self.no_positive and not positive_ids: + raise ValueError("Mind2Web row requires positive_ids or no_positive=true") + + object.__setattr__(self, "example_id", example_id) + object.__setattr__(self, "candidate_ids", candidate_ids) + object.__setattr__(self, "target_id", target_id or (positive_ids[0] if positive_ids else None)) + object.__setattr__(self, "positive_ids", positive_ids) + for field_name in ("task_id", "action_id", "split", "website", "domain"): + value = getattr(self, field_name) + object.__setattr__(self, field_name, None if value is None else str(value)) + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> Mind2WebExample: + if not isinstance(value, Mapping): + raise TypeError("Mind2Web row must be an object") + candidates = _string_sequence(value.get("candidate_ids", value.get("candidates", ())), "candidates") + positive_value = value.get("positive_ids", value.get("target_ids", value.get("targets"))) + legacy_target = value.get("target_id") + no_positive = value.get("no_positive", False) + if not isinstance(no_positive, bool): + raise TypeError("Mind2Web no_positive must be a boolean") + + if positive_value is None and legacy_target not in (None, ""): + positive_ids = (str(legacy_target),) + elif positive_value is None: + positive_ids = () + else: + positive_ids = _string_sequence(positive_value, "positive_ids") + if legacy_target not in (None, "") and str(legacy_target) not in positive_ids: + raise ValueError("target_id must be one of positive_ids") + + if not value.get("id"): + raise ValueError("Mind2Web row requires an id") + if len(candidates) != len(set(candidates)): + raise ValueError("Mind2Web candidates must be unique") + if not positive_ids and not no_positive: + raise ValueError("Mind2Web row requires positive_ids or no_positive=true") + + return cls( + example_id=str(value["id"]), + candidate_ids=candidates, + target_id=positive_ids[0] if positive_ids else None, + positive_ids=positive_ids, + no_positive=no_positive, + task_id=_optional_string(value.get("task_id")), + action_id=_optional_string(value.get("action_id")), + split=_optional_string(value.get("split")), + website=_optional_string(value.get("website")), + domain=_optional_string(value.get("domain")), + ) + + +def _optional_string(value: Any) -> str | None: + return None if value is None else str(value) + + +def _string_sequence(value: Any, label: str) -> tuple[str, ...]: + if isinstance(value, (str, bytes, bytearray)) or not isinstance(value, Sequence): + raise TypeError(f"Mind2Web {label} must be a list") + values: list[str] = [] + for item in value: + candidate = item.get("id") if isinstance(item, Mapping) else item + if not isinstance(candidate, str) or not candidate: + raise TypeError(f"Mind2Web {label} ids must be non-empty strings") + values.append(candidate) + return tuple(values) + + +@dataclass(frozen=True) +class Mind2WebMetrics: + rows: int + candidate_recall: float | None + selection_accuracy_given_recall: float | None + complete_case_selection_accuracy_given_recall: float | None + evaluated_recalled_rows: int + positive_rows: int + no_positive_rows: int + recalled_positive_rows: int + correct_selections: int + missing_predictions: int + invalid_selections: int + k: int | None = None + candidate_recall_task_macro: float | None = None + selection_accuracy_given_recall_task_macro: float | None = None + task_group_count: int = 0 + + def to_mapping(self) -> dict[str, int | float | str | None]: + return { + "rows": self.rows, + "candidate_recall": self.candidate_recall, + "selection_accuracy_given_recall": self.selection_accuracy_given_recall, + "complete_case_selection_accuracy_given_recall": self.complete_case_selection_accuracy_given_recall, + "evaluated_recalled_rows": self.evaluated_recalled_rows, + "positive_rows": self.positive_rows, + "no_positive_rows": self.no_positive_rows, + "recalled_positive_rows": self.recalled_positive_rows, + "correct_selections": self.correct_selections, + "missing_predictions": self.missing_predictions, + "invalid_selections": self.invalid_selections, + "k": self.k, + "candidate_recall_step_micro": self.candidate_recall, + "selection_accuracy_given_recall_step_micro": self.selection_accuracy_given_recall, + "candidate_recall_task_macro": self.candidate_recall_task_macro, + "selection_accuracy_given_recall_task_macro": self.selection_accuracy_given_recall_task_macro, + "task_group_count": self.task_group_count, + "scope": "candidate recall and candidate-in-set selection only; no browser task success", + } + + +def evaluate_mind2web( + examples: Sequence[Mind2WebExample], + generated_candidates: Mapping[str, Sequence[str]], + selections: Mapping[str, str | None], + *, + k: int | None = None, +) -> Mind2WebMetrics: + if k is not None and k <= 0: + raise ValueError("k must be positive") + positive_rows = sum(not row.no_positive for row in examples) + no_positive_rows = sum(row.no_positive for row in examples) + recalled = 0 + selected_correct = 0 + evaluated_recalled_rows = 0 + missing_predictions = 0 + invalid_selections = 0 + task_groups: dict[str, dict[str, int]] = {} + for row in examples: + group_key = row.task_id or row.example_id + group = task_groups.setdefault(group_key, {"positive": 0, "recalled": 0, "correct": 0}) + if not row.no_positive: + group["positive"] += 1 + candidates = tuple(str(item) for item in generated_candidates.get(row.example_id, ())) + if k is not None: + candidates = candidates[:k] + if row.no_positive or not set(row.positive_ids).intersection(candidates): + continue + recalled += 1 + group["recalled"] += 1 + selection = selections.get(row.example_id) + if selection is None: + missing_predictions += 1 + continue + evaluated_recalled_rows += 1 + if str(selection) not in candidates: + invalid_selections += 1 + continue + if str(selection) in row.positive_ids: + selected_correct += 1 + group["correct"] += 1 + task_recall_values = [ + group["recalled"] / group["positive"] + for group in task_groups.values() + if group["positive"] + ] + task_selection_values = [ + group["correct"] / group["recalled"] + for group in task_groups.values() + if group["recalled"] + ] + return Mind2WebMetrics( + rows=len(examples), + candidate_recall=recalled / positive_rows if positive_rows else None, + selection_accuracy_given_recall=selected_correct / recalled if recalled else None, + complete_case_selection_accuracy_given_recall=( + selected_correct / evaluated_recalled_rows if evaluated_recalled_rows else None + ), + evaluated_recalled_rows=evaluated_recalled_rows, + positive_rows=positive_rows, + no_positive_rows=no_positive_rows, + recalled_positive_rows=recalled, + correct_selections=selected_correct, + missing_predictions=missing_predictions, + invalid_selections=invalid_selections, + k=k, + candidate_recall_task_macro=( + sum(task_recall_values) / len(task_recall_values) if task_recall_values else None + ), + selection_accuracy_given_recall_task_macro=( + sum(task_selection_values) / len(task_selection_values) if task_selection_values else None + ), + task_group_count=len(task_groups), + ) + + +def _percentile(values: Sequence[float], fraction: float) -> float: + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + if lower == upper: + return ordered[lower] + return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + + +def task_bootstrap_intervals( + examples: Sequence[Mind2WebExample], + generated_candidates: Mapping[str, Sequence[str]], + selections: Mapping[str, str | None], + *, + k: int | None = None, + draws: int = 1000, + seed: int = 7, +) -> dict[str, Any]: + """Bootstrap task-macro recall and conditional selection at a fixed k. + + Sampling is over task groups, not individual action rows. Missing and + invalid selections remain errors in the primary recalled-row denominator. + """ + if draws < 1: + raise ValueError("draws must be positive") + if k is not None and k <= 0: + raise ValueError("k must be positive") + groups: dict[str, dict[str, int]] = {} + for row in examples: + group = groups.setdefault(row.task_id or row.example_id, {"positive": 0, "recalled": 0, "correct": 0}) + if row.no_positive: + continue + group["positive"] += 1 + candidates = tuple(str(item) for item in generated_candidates.get(row.example_id, ())) + if k is not None: + candidates = candidates[:k] + if not set(row.positive_ids).intersection(candidates): + continue + group["recalled"] += 1 + selection = selections.get(row.example_id) + if selection is not None and str(selection) in candidates and str(selection) in row.positive_ids: + group["correct"] += 1 + + recall_values = [ + group["recalled"] / group["positive"] for group in groups.values() if group["positive"] + ] + selection_values = [ + group["correct"] / group["recalled"] for group in groups.values() if group["recalled"] + ] + rng = random.Random(seed) + + def interval(values: list[float]) -> list[float] | None: + if not values: + return None + samples = [sum(rng.choice(values) for _ in values) / len(values) for _ in range(draws)] + return [_percentile(samples, 0.025), _percentile(samples, 0.975)] + + return { + "unit": "task_id_or_example_id", + "task_group_count": len(groups), + "draws": draws, + "seed": seed, + "candidate_recall_task_macro_ci95": interval(recall_values), + "selection_accuracy_given_recall_task_macro_ci95": interval(selection_values), + } + + +def recall_at_k( + examples: Sequence[Mind2WebExample], + generated_candidates: Mapping[str, Sequence[str]], + k: int, +) -> float | None: + """Return positive-row candidate recall at ``k``; undefined is ``None``.""" + + return evaluate_mind2web(examples, generated_candidates, {}, k=k).candidate_recall diff --git a/vons/models.py b/vons/models.py new file mode 100644 index 0000000000000000000000000000000000000000..5a64b68150359ad7c563652718ff1cd7311b68fb --- /dev/null +++ b/vons/models.py @@ -0,0 +1,102 @@ +"""Optional PyTorch Direct and Diffusion model implementations.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class ModelConfig: + hidden_size: int = 256 + latent_size: int = 32 + diffusion_steps: int = 32 + inference_steps: int = 4 + abstain_threshold: float = 0.55 + + +def require_torch() -> tuple[Any, Any]: + try: + import torch + from torch import nn + except ImportError as exc: + raise RuntimeError("training requires optional dependencies: pip install -e '.[train]'") from exc + return torch, nn + + +def build_model(backend: str, *, option_count: int, config: ModelConfig | None = None) -> Any: + """Build the decision head while keeping torch optional for data-only installs.""" + _torch, nn = require_torch() + if not 2 <= option_count <= 32: + raise ValueError("option_count must be between 2 and 32") + settings = config or ModelConfig() + + class DirectDecision(nn.Module): + def __init__(self) -> None: + super().__init__() + self.scorer = nn.Sequential(nn.Linear(settings.hidden_size, settings.hidden_size // 2), nn.GELU(), nn.Linear(settings.hidden_size // 2, 1)) + self.answerability = nn.Linear(settings.hidden_size, 1) + + def forward(self, option_embeddings: Any, pooled: Any, option_mask: Any | None = None) -> dict[str, Any]: + logits = self.scorer(option_embeddings).squeeze(-1) + if option_mask is not None: + logits = logits.masked_fill(~option_mask.bool(), float("-inf")) + return {"logits": logits, "answerability": self.answerability(pooled).squeeze(-1)} + + class DiffusionDecision(nn.Module): + def __init__(self) -> None: + super().__init__() + self.condition = nn.Linear(settings.hidden_size, settings.latent_size) + self.input = nn.Linear(option_count, settings.latent_size) + self.time = nn.Embedding(settings.diffusion_steps, settings.latent_size) + self.denoiser = nn.Sequential(nn.Linear(settings.latent_size, settings.hidden_size // 2), nn.SiLU(), nn.Linear(settings.hidden_size // 2, option_count)) + self.answerability = nn.Linear(settings.hidden_size, 1) + + def forward(self, noisy_scores: Any, pooled: Any, timestep: Any) -> dict[str, Any]: + latent = self.input(noisy_scores) + self.condition(pooled) + self.time(timestep) + return {"noise": self.denoiser(latent), "answerability": self.answerability(pooled).squeeze(-1)} + + return DirectDecision() if backend == "direct" else DiffusionDecision() + + +def linear_beta_schedule(steps: int) -> Any: + torch, _ = require_torch() + if steps < 2: + raise ValueError("diffusion schedule requires at least two steps") + return torch.linspace(1e-4, 0.02, steps) + + +def ddim_step(sample: Any, predicted_noise: Any, alpha: Any, alpha_previous: Any) -> Any: + """Deterministic DDIM update for a score vector.""" + torch, _ = require_torch() + if sample.shape != predicted_noise.shape: + raise ValueError("sample and predicted_noise must have the same shape") + predicted_x0 = (sample - torch.sqrt(1 - alpha) * predicted_noise) / torch.sqrt(alpha) + return torch.sqrt(alpha_previous) * predicted_x0 + torch.sqrt(1 - alpha_previous) * predicted_noise + + +def ddim_sample(model: Any, pooled: Any, *, option_count: int, diffusion_steps: int = 32, + inference_steps: int = 4, seed: int | None = None) -> Any: + """Sample candidate scores with deterministic DDIM updates. + + The returned scores are converted to probabilities by the caller so that + candidate masking and abstention remain visible at the contract boundary. + """ + torch, _ = require_torch() + if not 2 <= inference_steps <= diffusion_steps: + raise ValueError("inference_steps must be between 2 and diffusion_steps") + betas = linear_beta_schedule(diffusion_steps).to(pooled.device) + alpha_bars = torch.cumprod(1.0 - betas, dim=0) + generator = None + if seed is not None: + generator = torch.Generator(device=pooled.device) + generator.manual_seed(seed) + sample = torch.randn((pooled.shape[0], option_count), device=pooled.device, generator=generator) + timesteps = torch.linspace(diffusion_steps - 1, 0, inference_steps, device=pooled.device).round().long() + for index, timestep in enumerate(timesteps): + current_alpha = alpha_bars[timestep] + previous_alpha = alpha_bars[timesteps[index + 1]] if index + 1 < len(timesteps) else pooled.new_tensor(1.0) + timestep_batch = torch.full((pooled.shape[0],), int(timestep), dtype=torch.long, device=pooled.device) + predicted_noise = model(sample, pooled, timestep_batch)["noise"] + sample = ddim_step(sample, predicted_noise, current_alpha, previous_alpha) + return sample diff --git a/vons/ollama.py b/vons/ollama.py new file mode 100644 index 0000000000000000000000000000000000000000..6ef2b5194b9ee18339995289dc49dcb73ae6febf --- /dev/null +++ b/vons/ollama.py @@ -0,0 +1,133 @@ +"""Auditable Ollama adapter for local data generation and cross-review.""" + +from __future__ import annotations + +import json +import platform +import time +import urllib.request +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +from .data import Example + + +@dataclass(frozen=True) +class OllamaModelInfo: + tag: str + digest: str | None + details: Mapping[str, Any] + capabilities: tuple[str, ...] + license_present: bool + + +@dataclass(frozen=True) +class OllamaRecord: + model: str + prompt_kind: str + prompt: str + response: str | None + created_at: str + elapsed_ms: float + metadata: Mapping[str, Any] + error: str | None = None + + +class OllamaClient: + def __init__(self, host: str = "http://127.0.0.1:11434", timeout: float = 120.0) -> None: + self.host, self.timeout = host.rstrip("/"), timeout + + def _request(self, path: str, payload: Mapping[str, Any] | None = None) -> Any: + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request(self.host + path, data=data, headers={"Content-Type": "application/json"}, method="POST" if payload is not None else "GET") + with urllib.request.urlopen(request, timeout=self.timeout) as response: + return json.loads(response.read().decode("utf-8")) + + def version(self) -> str: + return str(self._request("/api/version")["version"]) + + def tags(self) -> list[Mapping[str, Any]]: + value = self._request("/api/tags") + models = value.get("models", []) if isinstance(value, Mapping) else [] + return [item for item in models if isinstance(item, Mapping)] + + def show(self, model: str) -> OllamaModelInfo: + value = self._request("/api/show", {"model": model}) + digest = value.get("digest") + if not digest: + digest = next((item.get("digest") for item in self.tags() if item.get("name") == model or item.get("model") == model), None) + return OllamaModelInfo(model, digest, dict(value.get("details", {})), tuple(value.get("capabilities", ())), bool(value.get("license"))) + + def chat_json(self, model: str, *, system: str, prompt: str, schema: Mapping[str, Any], seed: int = 7) -> OllamaRecord: + started = time.perf_counter() + created_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + body: Mapping[str, Any] = {} + try: + body = self._request("/api/chat", {"model": model, "messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}], "stream": False, "format": schema, "options": {"temperature": 0, "seed": seed}, "think": False}) + response = body.get("message", {}).get("content", "") + info = self.show(model) + error = None + except (OSError, ValueError, KeyError, RuntimeError, TypeError) as exc: + response, info, error = None, None, f"{type(exc).__name__}: {exc}" + return OllamaRecord(model, "chat_json", prompt, response, created_at, (time.perf_counter() - started) * 1000, {"host": self.host, "platform": platform.platform(), "ollama_version": self.version() if error is None else None, "digest": info.digest if info else None, "details": dict(info.details) if info else {}, "capabilities": list(info.capabilities) if info else [], "license_present": info.license_present if info else None, "raw_fields": sorted(body), "raw_response": body}, error) + + +def record_to_mapping(record: OllamaRecord) -> dict[str, Any]: + return asdict(record) + + +def benchmark_ollama(client: OllamaClient, model: str, examples: Sequence[Example], *, role: str = "judge", limit: int = 200, seed: int = 7) -> dict[str, Any]: + """Measure JSON compliance and judgment on a bounded, known-label set. + + The model's self-reported confidence is retained only as raw output. It is + never used as a probability or as a substitute for the dataset label. + """ + schema = { + "type": "object", + "properties": {"choice": {"type": ["string", "null"]}, "abstain": {"type": "boolean"}}, + "required": ["choice", "abstain"], + "additionalProperties": False, + } + system = "Return only JSON with a choice from the candidates or null and an abstain boolean. Do not call tools." + records: list[dict[str, Any]] = [] + for row in list(examples)[:limit]: + prompt = json.dumps({"state": row.state, "question": row.question, "candidates": list(row.options)}, ensure_ascii=False) + record = client.chat_json(model, system=system, prompt=prompt, schema=schema, seed=seed) + item = record_to_mapping(record) + item.update({"example_id": row.id, "role": role, "expected_label": row.label, "expected_answerable": row.answerable}) + parsed: Mapping[str, Any] | None = None + valid_json = False + if record.response: + try: + candidate = json.loads(record.response) + if isinstance(candidate, Mapping) and isinstance(candidate.get("abstain"), bool): + choice = candidate.get("choice") + valid_json = choice is None or isinstance(choice, str) + parsed = candidate if valid_json else None + except json.JSONDecodeError: + parsed = None + item["parsed"] = dict(parsed) if parsed is not None else None + item["valid_json"] = valid_json + item["correct"] = bool(valid_json and ((row.answerable and parsed and parsed.get("choice") == row.label and not parsed.get("abstain")) or (not row.answerable and parsed and parsed.get("abstain")))) + records.append(item) + valid = [item for item in records if item["valid_json"]] + correct = [item for item in records if item["correct"]] + latencies = sorted(float(item["elapsed_ms"]) for item in records) + return { + "model": model, + "role": role, + "seed": seed, + "rows": len(records), + "json_valid_rate": len(valid) / len(records) if records else 0.0, + "judgment_accuracy": len(correct) / len(records) if records else 0.0, + "latency_p50_ms": latencies[len(latencies) // 2] if latencies else None, + "records": records, + } + + +def write_benchmark(path: str | Path, result: Mapping[str, Any]) -> None: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(dict(result), ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") diff --git a/vons/research.py b/vons/research.py new file mode 100644 index 0000000000000000000000000000000000000000..49c7605243be9d232e14405d5a76a0880c8de397 --- /dev/null +++ b/vons/research.py @@ -0,0 +1,50 @@ +"""Build evidence packets for Claude Science and public research artifacts.""" + +from __future__ import annotations + +import hashlib +import json +import platform +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def collect_evidence(root: str | Path) -> dict[str, Any]: + directory = Path(root) + files = [] + if directory.exists(): + for path in sorted(directory.rglob("*")): + if path.is_file() and path.stat().st_size < 25 * 1024 * 1024: + files.append({"path": str(path), "bytes": path.stat().st_size, "sha256": _sha256_file(path)}) + try: + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=directory, text=True, stderr=subprocess.DEVNULL).strip() + except (OSError, subprocess.SubprocessError): + revision = None + return {"created_at": datetime.now(timezone.utc).isoformat(), "python": sys.version, "platform": platform.platform(), "git_revision": revision, "evidence_root": str(directory), "files": files} + + +def build_research_packet(*, results_dir: str | Path, output: str | Path, project: str = "Vons") -> Path: + packet = { + "project": project, + "purpose": "Prepare evidence for Claude Science paper and Tech Report drafting.", + "paper_title": "Vons: Compact Decision Models for Frontier-Agent Workflows", + "documents": ["paper", "tech_report"], + "claim_policy": {"measured": "Only values in result JSON and linked raw evidence.", "unmeasured": "Keep as TBD; never infer from design targets.", "confidence": "Do not treat model self-reported confidence as probability."}, + "evidence": collect_evidence(results_dir), + "claude_science_prompt": "Draft an evidence-bound English document for Vons. Separate measured, simulated, design-target, and unavailable values. Do not invent benchmark results and list unresolved limitations.", + } + destination = Path(output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(packet, indent=2, sort_keys=True), encoding="utf-8") + return destination diff --git a/vons/training.py b/vons/training.py new file mode 100644 index 0000000000000000000000000000000000000000..ad9feb1e4851add567fab1d78538c9476f654d43 --- /dev/null +++ b/vons/training.py @@ -0,0 +1,527 @@ +"""Small reproducible training/evaluation/export runners. + +The encoder is deliberately loaded from an explicit Hugging Face revision. The +decision heads stay in this repository so Direct and Diffusion share the same +candidate representation during the first comparison. +""" + +from __future__ import annotations + +import json +import platform +import random +import time +from pathlib import Path +from typing import Any + +from .data import Example, dataset_manifest, read_jsonl +from .evaluation import Prediction, summarize, write_report +from .models import ModelConfig, build_model, ddim_sample, require_torch + + +def _require_transformers() -> tuple[Any, Any]: + try: + from transformers import AutoModel, AutoTokenizer + except ImportError as exc: + raise RuntimeError("training requires optional dependencies: pip install -e '.[train]'") from exc + return AutoModel, AutoTokenizer + + +def _load_config(path: str | Path) -> dict[str, Any]: + value = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("training config must be a JSON object") + return value + + +def _device(torch: Any, requested: str) -> Any: + if requested != "auto": + return torch.device(requested) + if torch.backends.mps.is_available(): + return torch.device("mps") + if torch.cuda.is_available(): + return torch.device("cuda") + return torch.device("cpu") + + +def _encode_examples(tokenizer: Any, encoder: Any, examples: list[Example], device: Any, max_length: int = 512) -> tuple[Any, Any, Any, Any, list[Example]]: + torch, _ = require_torch() + option_embeddings: list[Any] = [] + pooled_embeddings: list[Any] = [] + targets: list[int] = [] + kept: list[Example] = [] + with torch.no_grad(): + for row in examples: + texts = [f"{row.state}\nQuestion: {row.question}\nCandidate: {option}" for option in row.options] + batch = tokenizer(texts, padding=True, truncation=True, max_length=max_length, return_tensors="pt").to(device) + output = encoder(**batch) + hidden = output.last_hidden_state[:, 0, :] + option_embeddings.append(hidden) + pooled_embeddings.append(hidden.mean(dim=0)) + targets.append(-1 if row.label is None else row.options.index(row.label)) + kept.append(row) + max_options = max((len(item.options) for item in kept), default=2) + hidden_size = option_embeddings[0].shape[-1] if option_embeddings else getattr(encoder.config, "hidden_size", 256) + padded = torch.zeros((len(kept), max_options, hidden_size), device=device) + mask = torch.zeros((len(kept), max_options), dtype=torch.bool, device=device) + pooled = torch.stack(pooled_embeddings) if pooled_embeddings else torch.empty((0, hidden_size), device=device) + for index, values in enumerate(option_embeddings): + padded[index, : values.shape[0]] = values + mask[index, : values.shape[0]] = True + return padded, pooled, torch.tensor(targets, dtype=torch.long, device=device), mask, kept + + +def _pad_option_axis(option_embeddings: Any, option_mask: Any, option_count: int) -> tuple[Any, Any]: + if option_embeddings.shape[1] > option_count: + raise ValueError(f"data contains {option_embeddings.shape[1]} candidates but option_count is {option_count}") + if option_embeddings.shape[1] == option_count: + return option_embeddings, option_mask + padded_embeddings = option_embeddings.new_zeros((option_embeddings.shape[0], option_count, option_embeddings.shape[-1])) + padded_mask = option_mask.new_zeros((option_mask.shape[0], option_count)) + padded_embeddings[:, : option_embeddings.shape[1]] = option_embeddings + padded_mask[:, : option_mask.shape[1]] = option_mask + return padded_embeddings, padded_mask + + +def train(config_path: str | Path) -> Path: + config = _load_config(config_path) + torch, nn = require_torch() + AutoModel, AutoTokenizer = _require_transformers() + seed = int(config.get("seed", 7)) + random.seed(seed) + torch.manual_seed(seed) + data_path = Path(config["data"]["path"]) + examples = read_jsonl(data_path) + train_rows = [row for row in examples if row.split == "train"] + if not train_rows: + raise ValueError("training data has no train split") + encoder_config = config["encoder"] + tokenizer = AutoTokenizer.from_pretrained(encoder_config["name"], revision=encoder_config.get("revision")) + encoder = AutoModel.from_pretrained(encoder_config["name"], revision=encoder_config.get("revision")) + device = _device(torch, str(config.get("device", "auto"))) + encoder.to(device).eval() + option_embeddings, pooled, targets, option_mask, kept = _encode_examples(tokenizer, encoder, train_rows, device, int(encoder_config.get("max_tokens", 512))) + option_count = int(config.get("option_count", option_embeddings.shape[1])) + option_embeddings, option_mask = _pad_option_axis(option_embeddings, option_mask, option_count) + backend = str(config.get("backend", "direct")) + model = build_model( + backend, + option_count=option_embeddings.shape[1], + config=ModelConfig( + hidden_size=option_embeddings.shape[-1], + diffusion_steps=int(config.get("diffusion_steps", 32)), + inference_steps=int(config.get("inference_steps", 4)), + abstain_threshold=float(config.get("abstain_threshold", 0.55)), + ), + ) + model.to(device).train() + optimizer = torch.optim.AdamW(model.parameters(), lr=float(config.get("learning_rate", 5e-5))) + epochs = int(config.get("epochs", 3)) + labels = targets.clamp_min(0) + answerable_targets = torch.tensor([row.answerable for row in kept], dtype=torch.float32, device=device) + valid = targets >= 0 + if not bool(valid.any()): + raise ValueError("training data has no answerable examples") + schedule = None + if backend == "diffusion": + from .models import linear_beta_schedule + + diffusion_steps = int(config.get("diffusion_steps", ModelConfig().diffusion_steps)) + betas = linear_beta_schedule(diffusion_steps).to(device) + schedule = torch.cumprod(1.0 - betas, dim=0) + batch_size = max(1, int(config.get("batch_size", 16))) + for _ in range(epochs): + permutation = torch.randperm(len(kept), device=device) + for start in range(0, len(kept), batch_size): + indices = permutation[start : start + batch_size] + batch_labels = labels[indices] + batch_valid = valid[indices] + optimizer.zero_grad() + if backend == "direct": + output = model(option_embeddings[indices], pooled[indices], option_mask[indices]) + if bool(batch_valid.any()): + decision_loss = nn.functional.cross_entropy(output["logits"][batch_valid], batch_labels[batch_valid]) + else: + decision_loss = output["answerability"].sum() * 0.0 + else: + target = torch.nn.functional.one_hot(batch_labels, num_classes=option_embeddings.shape[1]).float() + timestep = torch.randint(0, len(schedule), (len(indices),), device=device) + noise = torch.randn_like(target) + alpha_bar = schedule[timestep].unsqueeze(-1) + noisy = alpha_bar.sqrt() * target + (1.0 - alpha_bar).sqrt() * noise + output = model(noisy, pooled[indices], timestep) + if bool(batch_valid.any()): + decision_loss = nn.functional.mse_loss(output["noise"][batch_valid], noise[batch_valid]) + else: + decision_loss = output["answerability"].sum() * 0.0 + answerability_loss = nn.functional.binary_cross_entropy_with_logits(output["answerability"], answerable_targets[indices]) + loss = decision_loss + 0.1 * answerability_loss + loss.backward() + optimizer.step() + artifact_dir = Path(config.get("artifact_dir", "artifacts/pilot")) + artifact_dir.mkdir(parents=True, exist_ok=True) + checkpoint = artifact_dir / "model.pt" + torch.save({"state_dict": model.state_dict(), "backend": backend, "option_count": int(option_embeddings.shape[1]), "hidden_size": int(option_embeddings.shape[-1]), "config": config}, checkpoint) + manifest = {"model_id": config.get("model_id", "vons-pilot"), "backend": backend, "encoder": encoder_config, "device": str(device), "platform": platform.platform(), "seed": seed, "rows": len(kept), "data": dataset_manifest(data_path, examples), "checkpoint": str(checkpoint), "created_at": time.time()} + (artifact_dir / "model-manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + return checkpoint + + +def evaluate(checkpoint_path: str | Path, input_path: str | Path, output_path: str | Path) -> Path: + torch, _ = require_torch() + AutoModel, AutoTokenizer = _require_transformers() + checkpoint = torch.load(checkpoint_path, map_location="cpu") + config = checkpoint["config"] + encoder_config = config["encoder"] + tokenizer = AutoTokenizer.from_pretrained(encoder_config["name"], revision=encoder_config.get("revision")) + encoder = AutoModel.from_pretrained(encoder_config["name"], revision=encoder_config.get("revision")) + encoder.eval() + rows = read_jsonl(input_path) + option_embeddings, pooled, _targets, option_mask, kept = _encode_examples(tokenizer, encoder, rows, torch.device("cpu"), int(encoder_config.get("max_tokens", 512))) + expected_options = int(checkpoint["option_count"]) + if option_embeddings.shape[1] > expected_options: + raise ValueError(f"evaluation input has {option_embeddings.shape[1]} candidates but checkpoint supports {expected_options}") + if option_embeddings.shape[1] < expected_options: + option_embeddings, option_mask = _pad_option_axis(option_embeddings, option_mask, expected_options) + model = build_model( + checkpoint["backend"], + option_count=checkpoint["option_count"], + config=ModelConfig( + hidden_size=checkpoint["hidden_size"], + diffusion_steps=int(config.get("diffusion_steps", 32)), + inference_steps=int(config.get("inference_steps", 4)), + abstain_threshold=float(config.get("abstain_threshold", 0.55)), + ), + ) + model.load_state_dict(checkpoint["state_dict"]) + model.eval() + predictions: list[Prediction] = [] + with torch.no_grad(): + for index, row in enumerate(kept): + started = time.perf_counter() + row_embeddings = option_embeddings[index : index + 1] + row_pooled = pooled[index : index + 1] + row_mask = option_mask[index : index + 1] + if checkpoint["backend"] == "direct": + output = model(row_embeddings, row_pooled, row_mask) + logits = output["logits"] + answerability = torch.sigmoid(output["answerability"])[0] + else: + logits = ddim_sample( + model, + row_pooled, + option_count=expected_options, + diffusion_steps=int(config.get("diffusion_steps", 32)), + inference_steps=int(config.get("inference_steps", 4)), + seed=None if config.get("seed") is None else int(config["seed"]) + index, + ) + answerability = torch.sigmoid(model( + torch.zeros_like(logits), row_pooled, + torch.zeros((1,), dtype=torch.long), + )["answerability"])[0] + logits = logits.masked_fill(~row_mask, float("-inf")) + probabilities = torch.softmax(logits, dim=-1) + values = probabilities[0, : len(row.options)] + choice_index = int(values.argmax()) + confidence = float(values.max() * answerability) + abstained = confidence < float(config.get("abstain_threshold", 0.55)) or float(answerability) < float(config.get("answerability_threshold", 0.5)) + predictions.append(Prediction(row.id, None if abstained else row.options[choice_index], {option: float(values[offset]) for offset, option in enumerate(row.options)}, confidence, abstained, (time.perf_counter() - started) * 1000)) + report = Path(output_path) + report_config = {**config, "latency_scope": "decision_head_only; encoder_embeddings_precomputed"} + write_report(report, config=report_config, summary=summarize(kept, predictions), predictions=predictions) + return report + + +def export_head(checkpoint_path: str | Path, output_path: str | Path) -> Path: + """Export a fixed candidate-count decision head; encoder export is separate.""" + torch, _ = require_torch() + checkpoint = torch.load(checkpoint_path, map_location="cpu") + config = checkpoint["config"] + model = build_model( + checkpoint["backend"], + option_count=checkpoint["option_count"], + config=ModelConfig( + hidden_size=checkpoint["hidden_size"], + diffusion_steps=int(config.get("diffusion_steps", 32)), + inference_steps=int(config.get("inference_steps", 4)), + ), + ) + model.load_state_dict(checkpoint["state_dict"]) + model.eval() + + class DirectExportWrapper(torch.nn.Module): + def __init__(self, head: Any) -> None: + super().__init__() + self.head = head + + def forward(self, option_embeddings: Any, pooled: Any, option_mask: Any) -> tuple[Any, Any]: + result = self.head(option_embeddings, pooled, option_mask) + return result["logits"], result["answerability"] + + class DiffusionExportWrapper(torch.nn.Module): + def __init__(self, head: Any) -> None: + super().__init__() + self.head = head + + def forward(self, noisy_scores: Any, pooled: Any, timestep: Any) -> tuple[Any, Any]: + result = self.head(noisy_scores, pooled, timestep) + return result["noise"], result["answerability"] + + if checkpoint["backend"] == "direct": + wrapper = DirectExportWrapper(model).eval() + inputs = ( + torch.zeros((1, checkpoint["option_count"], checkpoint["hidden_size"])), + torch.zeros((1, checkpoint["hidden_size"])), + torch.ones((1, checkpoint["option_count"]), dtype=torch.bool), + ) + input_names = ["option_embeddings", "pooled", "option_mask"] + output_names = ["logits", "answerability"] + dynamic_axes = { + "option_embeddings": {0: "batch"}, "pooled": {0: "batch"}, "option_mask": {0: "batch"}, + "logits": {0: "batch"}, "answerability": {0: "batch"}, + } + elif checkpoint["backend"] == "diffusion": + wrapper = DiffusionExportWrapper(model).eval() + inputs = ( + torch.zeros((1, checkpoint["option_count"])), + torch.zeros((1, checkpoint["hidden_size"])), + torch.zeros((1,), dtype=torch.long), + ) + input_names = ["noisy_scores", "pooled", "timestep"] + output_names = ["noise", "answerability"] + dynamic_axes = { + "noisy_scores": {0: "batch"}, "pooled": {0: "batch"}, "timestep": {0: "batch"}, + "noise": {0: "batch"}, "answerability": {0: "batch"}, + } + else: + raise ValueError(f"unsupported backend {checkpoint['backend']!r}") + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + torch.onnx.export( + wrapper, + inputs, + output, + input_names=input_names, + output_names=output_names, + dynamic_axes=dynamic_axes, + opset_version=18, + ) + return output + + +def export_direct_bundle(checkpoint_path: str | Path, output_path: str | Path) -> Path: + """Export the Direct encoder and decision head as one ONNX graph. + + The graph keeps candidate count dynamic but preserves the same per-candidate + BERT CLS representation used during training. Tokenizer files are copied + beside the graph by the caller; this function does not invent token IDs or + a browser runtime. + """ + torch, _ = require_torch() + AutoModel, AutoTokenizer = _require_transformers() + checkpoint = torch.load(checkpoint_path, map_location="cpu") + if checkpoint["backend"] != "direct": + raise ValueError("end-to-end bundle export currently supports the direct backend only") + config = checkpoint["config"] + encoder_config = config["encoder"] + encoder = AutoModel.from_pretrained(encoder_config["name"], revision=encoder_config.get("revision")) + encoder.eval() + model = build_model( + "direct", + option_count=checkpoint["option_count"], + config=ModelConfig(hidden_size=checkpoint["hidden_size"]), + ) + model.load_state_dict(checkpoint["state_dict"]) + model.eval() + + class FullDirectExportWrapper(torch.nn.Module): + def __init__(self, encoder_model: Any, head: Any) -> None: + super().__init__() + self.encoder = encoder_model + self.head = head + + def forward(self, input_ids: Any, attention_mask: Any, token_type_ids: Any, option_mask: Any) -> tuple[Any, Any]: + batch, options, sequence = input_ids.shape + flat_ids = input_ids.reshape(batch * options, sequence) + flat_attention = attention_mask.reshape(batch * options, sequence) + flat_types = token_type_ids.reshape(batch * options, sequence) + encoded = self.encoder( + input_ids=flat_ids, + attention_mask=flat_attention, + token_type_ids=flat_types, + ).last_hidden_state[:, 0, :] + candidate_embeddings = encoded.reshape(batch, options, -1) + weights = option_mask.to(candidate_embeddings.dtype).unsqueeze(-1) + pooled = (candidate_embeddings * weights).sum(dim=1) / weights.sum(dim=1).clamp_min(1) + result = self.head(candidate_embeddings, pooled, option_mask) + return result["logits"], result["answerability"] + + wrapper = FullDirectExportWrapper(encoder, model).eval() + sequence = int(encoder_config.get("max_tokens", 512)) + option_count = int(checkpoint["option_count"]) + inputs = ( + torch.zeros((1, option_count, sequence), dtype=torch.long), + torch.ones((1, option_count, sequence), dtype=torch.long), + torch.zeros((1, option_count, sequence), dtype=torch.long), + torch.ones((1, option_count), dtype=torch.bool), + ) + input_names = ["input_ids", "attention_mask", "token_type_ids", "option_mask"] + output_names = ["logits", "answerability"] + dynamic_axes = { + name: {0: "batch"} for name in input_names + output_names + } + dynamic_axes["input_ids"][1] = "options" + dynamic_axes["input_ids"][2] = "sequence" + dynamic_axes["attention_mask"][1] = "options" + dynamic_axes["attention_mask"][2] = "sequence" + dynamic_axes["token_type_ids"][1] = "options" + dynamic_axes["token_type_ids"][2] = "sequence" + dynamic_axes["option_mask"][1] = "options" + dynamic_axes["logits"][1] = "options" + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + torch.onnx.export( + wrapper, + inputs, + output, + input_names=input_names, + output_names=output_names, + dynamic_axes=dynamic_axes, + opset_version=18, + ) + tokenizer = AutoTokenizer.from_pretrained(encoder_config["name"], revision=encoder_config.get("revision")) + tokenizer.save_pretrained(output.parent / "tokenizer") + manifest = { + "format": "onnx", + "backend": "direct", + "encoder": encoder_config, + "checkpoint": str(checkpoint_path), + "onnx": str(output), + "tokenizer": str(output.parent / "tokenizer"), + "option_count": option_count, + "sequence_length": sequence, + "input_names": input_names, + "output_names": output_names, + "scope": "end_to_end_encoder_plus_decision_head", + } + (output.parent / "bundle-manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + return output + + +def export_diffusion_bundle(checkpoint_path: str | Path, output_path: str | Path) -> Path: + """Export a fixed-step Diffusion encoder/head graph as one ONNX bundle.""" + torch, _ = require_torch() + AutoModel, AutoTokenizer = _require_transformers() + checkpoint = torch.load(checkpoint_path, map_location="cpu") + if checkpoint["backend"] != "diffusion": + raise ValueError("diffusion bundle export requires a diffusion checkpoint") + config = checkpoint["config"] + encoder_config = config["encoder"] + encoder = AutoModel.from_pretrained(encoder_config["name"], revision=encoder_config.get("revision")) + encoder.eval() + diffusion_steps = int(config.get("diffusion_steps", 32)) + inference_steps = int(config.get("inference_steps", 4)) + if not 2 <= inference_steps <= diffusion_steps: + raise ValueError("inference_steps must be between 2 and diffusion_steps") + model = build_model( + "diffusion", + option_count=checkpoint["option_count"], + config=ModelConfig( + hidden_size=checkpoint["hidden_size"], + diffusion_steps=diffusion_steps, + inference_steps=inference_steps, + ), + ) + model.load_state_dict(checkpoint["state_dict"]) + model.eval() + + class FullDiffusionExportWrapper(torch.nn.Module): + def __init__(self, encoder_model: Any, head: Any) -> None: + super().__init__() + self.encoder = encoder_model + self.head = head + betas = torch.linspace(1e-4, 0.02, diffusion_steps) + self.register_buffer("alpha_bars", torch.cumprod(1.0 - betas, dim=0)) + + def forward( + self, + input_ids: Any, + attention_mask: Any, + token_type_ids: Any, + option_mask: Any, + initial_noise: Any, + ) -> tuple[Any, Any]: + batch, options, sequence = input_ids.shape + flat_ids = input_ids.reshape(batch * options, sequence) + flat_attention = attention_mask.reshape(batch * options, sequence) + flat_types = token_type_ids.reshape(batch * options, sequence) + encoded = self.encoder( + input_ids=flat_ids, + attention_mask=flat_attention, + token_type_ids=flat_types, + ).last_hidden_state[:, 0, :] + candidate_embeddings = encoded.reshape(batch, options, -1) + weights = option_mask.to(candidate_embeddings.dtype).unsqueeze(-1) + pooled = (candidate_embeddings * weights).sum(dim=1) / weights.sum(dim=1).clamp_min(1) + sample = initial_noise + timesteps = torch.linspace(diffusion_steps - 1, 0, inference_steps).round().to(torch.long) + for index in range(inference_steps): + timestep = timesteps[index] + timestep_batch = torch.full((batch,), timestep, dtype=torch.long, device=sample.device) + predicted_noise = self.head(sample, pooled, timestep_batch)["noise"] + alpha = self.alpha_bars[timestep] + previous_timestep = timesteps[index + 1] if index + 1 < inference_steps else torch.tensor(0, device=sample.device) + previous_alpha = self.alpha_bars[previous_timestep] if index + 1 < inference_steps else sample.new_tensor(1.0) + predicted_x0 = (sample - torch.sqrt(1 - alpha) * predicted_noise) / torch.sqrt(alpha) + sample = torch.sqrt(previous_alpha) * predicted_x0 + torch.sqrt(1 - previous_alpha) * predicted_noise + sample = sample.masked_fill(~option_mask.bool(), float("-inf")) + answerability = self.head(initial_noise, pooled, torch.zeros((batch,), dtype=torch.long, device=sample.device))["answerability"] + return sample, answerability + + wrapper = FullDiffusionExportWrapper(encoder, model).eval() + sequence = int(encoder_config.get("max_tokens", 512)) + option_count = int(checkpoint["option_count"]) + inputs = ( + torch.zeros((1, option_count, sequence), dtype=torch.long), + torch.ones((1, option_count, sequence), dtype=torch.long), + torch.zeros((1, option_count, sequence), dtype=torch.long), + torch.ones((1, option_count), dtype=torch.bool), + torch.zeros((1, option_count)), + ) + input_names = ["input_ids", "attention_mask", "token_type_ids", "option_mask", "initial_noise"] + output_names = ["scores", "answerability"] + # Diffusion's first layer is parameterized by the configured candidate + # count, so its option and sequence dimensions are intentionally fixed. + # Batch remains dynamic for serving. + dynamic_axes = {name: {0: "batch"} for name in input_names + output_names} + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + torch.onnx.export( + wrapper, + inputs, + output, + input_names=input_names, + output_names=output_names, + dynamic_axes=dynamic_axes, + opset_version=18, + ) + tokenizer = AutoTokenizer.from_pretrained(encoder_config["name"], revision=encoder_config.get("revision")) + tokenizer.save_pretrained(output.parent / "tokenizer") + manifest = { + "format": "onnx", + "backend": "diffusion", + "encoder": encoder_config, + "checkpoint": str(checkpoint_path), + "onnx": str(output), + "tokenizer": str(output.parent / "tokenizer"), + "option_count": option_count, + "sequence_length": sequence, + "diffusion_steps": diffusion_steps, + "inference_steps": inference_steps, + "input_names": input_names, + "output_names": output_names, + "scope": "end_to_end_encoder_plus_fixed_ddim_decision_head", + "seed_contract": "host supplies deterministic initial_noise", + } + (output.parent / "bundle-manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + return output