Mirror GitHub f31f2ceadd54
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +1 -0
- CITATION.cff +28 -0
- DATA_PROVENANCE.md +34 -0
- GITHUB_README.md +37 -0
- LICENSE +21 -0
- MANIFEST.json +473 -0
- PUBLICATION.md +20 -0
- README.md +19 -4
- RELEASE_SCOPE.txt +26 -0
- SNAPSHOT.json +75 -0
- code/canonical_readouts_v1.py +75 -0
- code/family_statistics.py +13 -0
- code/frozen/PBEBench_functional_correctness.py +525 -0
- code/frozen/PBEBench_primitives.py +118 -0
- code/frozen/PBEBench_utils.py +11 -0
- code/frozen/analyze_panel.py +83 -0
- code/frozen/functional_score.py +29 -0
- code/native_score.py +197 -0
- data/analysis_contract.json +52 -0
- data/base_variants.jsonl +0 -0
- data/broader_inputs.jsonl +0 -0
- data/certificate_records.jsonl +0 -0
- data/construction_audit.csv +481 -0
- data/families.jsonl +0 -0
- data/model_contract.json +184 -0
- data/native1008.csv +0 -0
- data/selected_programs.csv +0 -0
- data/synthesis_instruction.txt +13 -0
- data/trajectories.jsonl.gz +3 -0
- docs/asset-manifest.csv +39 -0
- docs/companion.json +282 -0
- docs/figures/construction.pdf +0 -0
- docs/figures/construction.png +3 -0
- docs/maintenance.md +5 -0
- docs/release-manifest.json +32 -0
- docs/report.en.md +67 -0
- docs/report.zh.md +67 -0
- docs/reproduction.md +27 -0
- docs/reproduction_report.json +76 -0
- docs/result-map.csv +11 -0
- explore.js +28 -0
- explorer.json +338 -0
- fixtures/end_to_end.json +230 -0
- index.html +18 -18
- licenses/CC-BY-4.0.txt +395 -0
- licenses/PBEBench-MIT.txt +21 -0
- program-structure-certificates-v1.0.0.zip +3 -0
- provenance/ANALYSIS_HISTORY.json +22 -0
- provenance/NORMALIZATION.json +29 -0
- replay_fixture.py +112 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
docs/figures/construction.png filter=lfs diff=lfs merge=lfs -text
|
CITATION.cff
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
cff-version: 1.2.0
|
| 2 |
+
message: "If you use the task data or CPU replay, cite this artifact and the accompanying paper."
|
| 3 |
+
title: "When Is Program Structure Required? Certificate-Grounded Calibration of Program-Induction Difficulty"
|
| 4 |
+
type: software
|
| 5 |
+
license: MIT
|
| 6 |
+
references:
|
| 7 |
+
- type: preprint
|
| 8 |
+
title: "When Is Program Structure Required? Certificate-Grounded Calibration of Program-Induction Difficulty"
|
| 9 |
+
authors:
|
| 10 |
+
- family-names: Du
|
| 11 |
+
given-names: Wenzhang
|
| 12 |
+
year: 2026
|
| 13 |
+
url: "https://arxiv.org/abs/2611.24975"
|
| 14 |
+
doi: "10.48550/arXiv.2611.24975"
|
| 15 |
+
authors:
|
| 16 |
+
- given-names: Wenzhang
|
| 17 |
+
family-names: Du
|
| 18 |
+
affiliation: "Independent Researcher, China"
|
| 19 |
+
version: 1.0.0
|
| 20 |
+
date-released: 2026-11-16
|
| 21 |
+
url: "https://github.com/dqswordman/program-structure-certificates/tree/v1.0.0"
|
| 22 |
+
repository-code: "https://github.com/dqswordman/program-structure-certificates"
|
| 23 |
+
keywords:
|
| 24 |
+
- program induction
|
| 25 |
+
- program synthesis
|
| 26 |
+
- benchmark calibration
|
| 27 |
+
- certificates
|
| 28 |
+
- reproducibility
|
DATA_PROVENANCE.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Data provenance
|
| 2 |
+
|
| 3 |
+
## Task and output records
|
| 4 |
+
|
| 5 |
+
| Component | Origin and coverage |
|
| 6 |
+
| --- | --- |
|
| 7 |
+
| task families, variants, and certificates | Study-generated bounded string transformations and independently checked minimum-length certificates. The feeding-only proposal grammar follows PBEBench. |
|
| 8 |
+
| evaluation inputs | Shared, wholly unshown, uniform, and rule-active strings with executable reference outputs. |
|
| 9 |
+
| normalized trajectories | The study's archived Qwen and Olmo evaluations, including generation text, token IDs, and parser outputs. See the model contract and normalization record. |
|
| 10 |
+
| retrospective archive | The saved 1,008 configurations and outputs used to audit one-rule shortcuts. Source row digests and the original reference descriptions travel with the archive. The upstream sampling and filtering chain for this saved roster is unresolved in the paper and archive. |
|
| 11 |
+
| historical results | Earlier evidence summarized in the paper and the analysis history record. The reproduction scope lists the underlying records available in the release. |
|
| 12 |
+
|
| 13 |
+
[Model contract](data/model_contract.json) · [Normalization record](provenance/NORMALIZATION.json) · [Retrospective archive](data/native1008.csv) · [Coverage](RELEASE_SCOPE.txt)
|
| 14 |
+
|
| 15 |
+
## Model revisions
|
| 16 |
+
|
| 17 |
+
| Model | Revision |
|
| 18 |
+
| --- | --- |
|
| 19 |
+
| Qwen3-8B | [b968826d9c46dd6066d109eabc6255188de91218](https://huggingface.co/Qwen/Qwen3-8B/tree/b968826d9c46dd6066d109eabc6255188de91218) |
|
| 20 |
+
| Olmo-3-7B-Think | [d97e442d7cc678210054dbcc9b440894d62c89a4](https://huggingface.co/allenai/Olmo-3-7B-Think/tree/d97e442d7cc678210054dbcc9b440894d62c89a4) |
|
| 21 |
+
|
| 22 |
+
The model contract preserves model and tokenizer revisions, chat template source, prompt provenance, and generation settings. Model weights are obtained from the respective publishers under their terms. The replay uses archived outputs.
|
| 23 |
+
|
| 24 |
+
## PBEBench attribution and reuse
|
| 25 |
+
|
| 26 |
+
The three PBEBench source files in code/frozen match the following upstream paths byte for byte at commit b2312327558e81c947ed8b543001169075062491.
|
| 27 |
+
|
| 28 |
+
| Included path | Upstream path |
|
| 29 |
+
| --- | --- |
|
| 30 |
+
| code/frozen/PBEBench_functional_correctness.py | src/metrics/functional_correctness.py |
|
| 31 |
+
| code/frozen/PBEBench_primitives.py | src/data_generation/primitives.py |
|
| 32 |
+
| code/frozen/PBEBench_utils.py | src/data_generation/utils.py |
|
| 33 |
+
|
| 34 |
+
The portable native scoring adapter retains the benchmark's scoring definitions. The upstream MIT copyright notice applies to the distributed PBEBench code and those adaptations. Original study software is MIT-licensed. Original task constructions, numerical annotations and companion materials are under CC BY 4.0. The historical 1,008-task roster retains its upstream provenance limits; the author's grant covers the author's audit measurements. Model-generated material, external templates and third-party components retain their applicable upstream terms. [Public-edition scope](PUBLICATION.md). [PBEBench code revision](https://github.com/cmu-llab/PBEBench/tree/b2312327558e81c947ed8b543001169075062491) · [MIT notice](licenses/PBEBench-MIT.txt) · [Benchmark paper](https://aclanthology.org/2026.findings-acl.432/)
|
GITHUB_README.md
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# When Is Program Structure Required? Certificate-Grounded Calibration of Program-Induction Difficulty
|
| 2 |
+
|
| 3 |
+
Wenzhang Du · Independent Researcher, China · arXiv preprint, November 2026
|
| 4 |
+
|
| 5 |
+
[Paper · arXiv:2611.24975](https://arxiv.org/abs/2611.24975) · [Technical report](https://dqswordman.github.io/technical-reports/program-structure-certificates/) · [中文伴读](https://dqswordman.github.io/zh/technical-reports/program-structure-certificates/) · [Interactive Space](https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates)
|
| 6 |
+
|
| 7 |
+
A generator can use two string-replacement rules even when the displayed examples admit a one-rule solution. This artifact follows a matched pair of specifications through complete one-rule enumeration, executable two-rule witnesses, and the programs returned by language models.
|
| 8 |
+
|
| 9 |
+
[Reproduction guide](docs/reproduction.md) · [Result map](docs/result-map.csv) · [Data provenance](DATA_PROVENANCE.md) · [Frozen artifact v1.0.0](https://github.com/dqswordman/program-structure-certificates/tree/v1.0.0)
|
| 10 |
+
|
| 11 |
+
## Follow one archived task pair
|
| 12 |
+
|
| 13 |
+
From the repository root with Python 3.11 or newer:
|
| 14 |
+
|
| 15 |
+
python -B verify_manifest.py
|
| 16 |
+
python -B replay_fixture.py
|
| 17 |
+
|
| 18 |
+
The fixture checks the six-example pair in Figure 1. The control admits the single replacement from exa to azy. The separation example excludes that rule, and the shared two-rule reference satisfies both specifications. The command reports certified minima of **one** and **two**, then executes four saved Qwen and Olmo programs on the displayed, witness, uniform, and rule-active inputs. It uses the included archive and the Python standard library.
|
| 19 |
+
|
| 20 |
+
[](docs/figures/construction.pdf)
|
| 21 |
+
|
| 22 |
+
## Results at a glance
|
| 23 |
+
|
| 24 |
+
Across both models and the three-to-six base-example settings, the certified separation tasks reduce displayed-task completion by **23.53 to 45.83 percentage points** and increase exact success on the wholly unshown witness pair by **8.40 to 13.45 points**. Agreement on strings activating both reference rules ranges from **1.93% to 5.78%** in the separation conditions. The archived tables distinguish these evaluation targets. [Primary panel](results/PRIMARY_PANEL_READABLE.csv) · [Broader-input panel](results/BROADER_PANEL_READABLE.csv)
|
| 25 |
+
|
| 26 |
+
## Replay the panel
|
| 27 |
+
|
| 28 |
+
Create an environment with NumPy, then run the full CPU check from the frozen tag:
|
| 29 |
+
|
| 30 |
+
python -m pip install -r requirements.txt
|
| 31 |
+
python -B reproduce.py --out replay
|
| 32 |
+
|
| 33 |
+
This reads **1,920 saved outputs**, checks **960 certificates** and **7,680 additional reference labels**, and regenerates the three comparison CSVs. The result map marks which historical analyses are available as aggregate tables and protocols. [Full instructions](docs/reproduction.md) · [Verification report](docs/reproduction_report.json)
|
| 34 |
+
|
| 35 |
+
## Citation
|
| 36 |
+
|
| 37 |
+
Cite Wenzhang Du, *When Is Program Structure Required? Certificate-Grounded Calibration of Program-Induction Difficulty*, arXiv:2611.24975 (2026). The [public release](PUBLICATION.md) lists the software and data licenses, the frozen revision and the [single offline download](https://github.com/dqswordman/program-structure-certificates/releases/download/v1.0.0/program-structure-certificates-v1.0.0.zip). [CITATION.cff](CITATION.cff) · [Version record](docs/release-manifest.json)
|
LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 Wenzhang Du
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
MANIFEST.json
ADDED
|
@@ -0,0 +1,473 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "program-structure-research-artifact-v1",
|
| 3 |
+
"title": "When Is Program Structure Required? Certificate-Grounded Calibration of Program-Induction Difficulty",
|
| 4 |
+
"version": "1.0.0",
|
| 5 |
+
"files": [
|
| 6 |
+
{
|
| 7 |
+
"path": "CITATION.cff",
|
| 8 |
+
"size_bytes": 987,
|
| 9 |
+
"sha256": "76529e711e08377121c1448bf8c577e6cf25d99e4d1eb464a174a14588009b9e"
|
| 10 |
+
},
|
| 11 |
+
{
|
| 12 |
+
"path": "DATA_PROVENANCE.md",
|
| 13 |
+
"size_bytes": 3134,
|
| 14 |
+
"sha256": "b0d2c322230982d5d289b12dedadc3eff4f0bf4396908ba117811b0c5923407f"
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"path": "GITHUB_README.md",
|
| 18 |
+
"size_bytes": 3430,
|
| 19 |
+
"sha256": "5d58fa73ea81a4a8bdec1892a43783d86439f95973aabfd52e6ff5dab48c3bf3"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"path": "LICENSE",
|
| 23 |
+
"size_bytes": 1068,
|
| 24 |
+
"sha256": "020893b24fb2a5b62d2d2748f65e569e31ab7b8eeb831b5e64bb7a57e544c104"
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"path": "PUBLICATION.md",
|
| 28 |
+
"size_bytes": 2652,
|
| 29 |
+
"sha256": "2011e8cf78f537ef0f42257d0703e5ccf7927ec943b7f5153ca20f3d6470f476"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"path": "README.md",
|
| 33 |
+
"size_bytes": 1337,
|
| 34 |
+
"sha256": "a26bbe7b397e2d5328857bb4122d6e215e372f7d9691de8121ffe631467dceb6"
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"path": "RELEASE_SCOPE.txt",
|
| 38 |
+
"size_bytes": 1547,
|
| 39 |
+
"sha256": "3701acb093ac3e9e9a348f9aa954d4a586d5c9defecc74a585aff430572ec6f1"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"path": "code/canonical_readouts_v1.py",
|
| 43 |
+
"size_bytes": 4513,
|
| 44 |
+
"sha256": "793af7b98682cb57b5efc47b5b534e86ef90725f7c3269b4fc098d76778126a6"
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"path": "code/family_statistics.py",
|
| 48 |
+
"size_bytes": 854,
|
| 49 |
+
"sha256": "2dd50ccb2d0e3819efa79e075573d8ce799ceb18011a9ba6229e0bc4bd095a83"
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"path": "code/frozen/PBEBench_functional_correctness.py",
|
| 53 |
+
"size_bytes": 24616,
|
| 54 |
+
"sha256": "db8ab01ef70be0c4ad34610045be4c6f6adad03977921efefc16260382901322"
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"path": "code/frozen/PBEBench_primitives.py",
|
| 58 |
+
"size_bytes": 4390,
|
| 59 |
+
"sha256": "9d68d96f6a86413f1f5849a4f589bc7ea009e5987555d7f14308bb851e92c043"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"path": "code/frozen/PBEBench_utils.py",
|
| 63 |
+
"size_bytes": 215,
|
| 64 |
+
"sha256": "418c27074bdcad4a3a3a1ff3a532b0ee7e9f46a2f5d41cf6da7908e5b8673b90"
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"path": "code/frozen/analyze_panel.py",
|
| 68 |
+
"size_bytes": 8111,
|
| 69 |
+
"sha256": "b6be13e25962d341bc24286c9d42b688487de49c62509598add793baa66d57aa"
|
| 70 |
+
},
|
| 71 |
+
{
|
| 72 |
+
"path": "code/frozen/functional_score.py",
|
| 73 |
+
"size_bytes": 1576,
|
| 74 |
+
"sha256": "b1fffcb1ec769693cd6100423d9ba3030ce1976674e62a1c9e0cb2cf6ccee904"
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"path": "code/native_score.py",
|
| 78 |
+
"size_bytes": 8581,
|
| 79 |
+
"sha256": "3174025e5904cbd7a5f5f31385facd6a2fb7e4c64d4f857a3e1381fe792c3425"
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"path": "data/analysis_contract.json",
|
| 83 |
+
"size_bytes": 2169,
|
| 84 |
+
"sha256": "4a6991cad6350c7c6fb29c5b0a9d8f0929f3e4b0149f9c05b720970b801ee1cd"
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"path": "data/base_variants.jsonl",
|
| 88 |
+
"size_bytes": 965314,
|
| 89 |
+
"sha256": "9cd4c40b7a1468a699c40a6aacda61e73e507868d2b9cf02ad98cbbda89006d1"
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"path": "data/broader_inputs.jsonl",
|
| 93 |
+
"size_bytes": 533575,
|
| 94 |
+
"sha256": "520948e896fa37c8d0389b035ad84afc9a741a20c9e3ecf8173f49771b421fe5"
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"path": "data/certificate_records.jsonl",
|
| 98 |
+
"size_bytes": 648598,
|
| 99 |
+
"sha256": "dab7802e8005f615c3b0f3c7dac92d34ffe547b2f9b8f55e1fcd101c4de5b850"
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"path": "data/construction_audit.csv",
|
| 103 |
+
"size_bytes": 49640,
|
| 104 |
+
"sha256": "de6d2a61a2b518f8d04b0351b290a19d684c16a704d9aa8eec6cb2bf5bbd9ff9"
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"path": "data/families.jsonl",
|
| 108 |
+
"size_bytes": 309171,
|
| 109 |
+
"sha256": "c6f7814d9085f597b81d310115cf42f03efe95f7c7cdec48cacdffb69b5198b7"
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"path": "data/model_contract.json",
|
| 113 |
+
"size_bytes": 11284,
|
| 114 |
+
"sha256": "b33554033c16b56605bd384a885001d052b3a0e346487ade40b4ff9ff354466a"
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"path": "data/native1008.csv",
|
| 118 |
+
"size_bytes": 387271,
|
| 119 |
+
"sha256": "76faa6b2f0f35874a840c11e1ef6cd3180d1665a856dc64e97293492a92a0343"
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"path": "data/selected_programs.csv",
|
| 123 |
+
"size_bytes": 656183,
|
| 124 |
+
"sha256": "94ca7ea378fce2954520f6641505440257bbb1529257f36ea66fc37dd4892e67"
|
| 125 |
+
},
|
| 126 |
+
{
|
| 127 |
+
"path": "data/synthesis_instruction.txt",
|
| 128 |
+
"size_bytes": 938,
|
| 129 |
+
"sha256": "4fb7efda45f0beb0978934612eeeca36e73da3d87f4977c17dd5f4d386318f11"
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"path": "data/trajectories.jsonl.gz",
|
| 133 |
+
"size_bytes": 26517321,
|
| 134 |
+
"sha256": "6d9c9af078d6ec25bb8f1e95894ed07dbd879c2e3d77b7dc5afe0081cf57ed7d"
|
| 135 |
+
},
|
| 136 |
+
{
|
| 137 |
+
"path": "docs/asset-manifest.csv",
|
| 138 |
+
"size_bytes": 6107,
|
| 139 |
+
"sha256": "99eff1d2a2930b2e6ed202c23c8884b7ccf308314437701ac65b51f28b0972c0"
|
| 140 |
+
},
|
| 141 |
+
{
|
| 142 |
+
"path": "docs/companion.json",
|
| 143 |
+
"size_bytes": 15203,
|
| 144 |
+
"sha256": "dbf82ff83cc84d124fa844ad26f1c406a1fb77c0dbb179a220bdc797c3b1624e"
|
| 145 |
+
},
|
| 146 |
+
{
|
| 147 |
+
"path": "docs/figures/construction.pdf",
|
| 148 |
+
"size_bytes": 83247,
|
| 149 |
+
"sha256": "1224ffa9a3dfa361a7661a9fea3763250414cf84380e6cb7dc4e0c211ac9396f"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"path": "docs/figures/construction.png",
|
| 153 |
+
"size_bytes": 140513,
|
| 154 |
+
"sha256": "73c9776e67c7fdb2524682b4a4759aa2a319843c7a667b2429cad369799291e1"
|
| 155 |
+
},
|
| 156 |
+
{
|
| 157 |
+
"path": "docs/maintenance.md",
|
| 158 |
+
"size_bytes": 480,
|
| 159 |
+
"sha256": "0cd2ddf55a1cd2694e53c65dbb5d71260038a761e9316c169ff2abe9f6ab9be4"
|
| 160 |
+
},
|
| 161 |
+
{
|
| 162 |
+
"path": "docs/release-manifest.json",
|
| 163 |
+
"size_bytes": 1289,
|
| 164 |
+
"sha256": "116d011a6508cd55eb58b5668823bbd7d79501c1834754973d3d9a6ed5877604"
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"path": "docs/report.en.md",
|
| 168 |
+
"size_bytes": 5955,
|
| 169 |
+
"sha256": "24d127dbe6800d8416b5dd35a53b6487b0faebba1caadb2d6e29d81de221a4e7"
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"path": "docs/report.zh.md",
|
| 173 |
+
"size_bytes": 5633,
|
| 174 |
+
"sha256": "d944b594e45668eae6b82b63de1e6e69b1831774b6c55fa7d47d88913b2331b3"
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"path": "docs/reproduction.md",
|
| 178 |
+
"size_bytes": 2195,
|
| 179 |
+
"sha256": "fa83a6b38ffa6a677dab1ec01a0a9ba7aff394a10ff9da14f1159f2069f83bac"
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"path": "docs/reproduction_report.json",
|
| 183 |
+
"size_bytes": 2525,
|
| 184 |
+
"sha256": "4aaac315cc92a9c0b51198d8fe7285243940155f0b61913d4b8951573a47711d"
|
| 185 |
+
},
|
| 186 |
+
{
|
| 187 |
+
"path": "docs/result-map.csv",
|
| 188 |
+
"size_bytes": 2223,
|
| 189 |
+
"sha256": "ac8a29e02c895deddfd10c5fdb9192c420fb8c32fb7d07f39c5f0eecd9ff2351"
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"path": "explore.js",
|
| 193 |
+
"size_bytes": 1624,
|
| 194 |
+
"sha256": "4e7a3d712bf7a7747ac7dacefe2e19e69de14afcb4ae826de163bd7703ff7e9e"
|
| 195 |
+
},
|
| 196 |
+
{
|
| 197 |
+
"path": "explorer.json",
|
| 198 |
+
"size_bytes": 10001,
|
| 199 |
+
"sha256": "29f6732ad3cc85e74378578b53bfdd1558de9a485349ec27dbbd1c4078676924"
|
| 200 |
+
},
|
| 201 |
+
{
|
| 202 |
+
"path": "fixtures/end_to_end.json",
|
| 203 |
+
"size_bytes": 5396,
|
| 204 |
+
"sha256": "de641ad2106e9b1fdc9e64214f0938a20e02f230ba4db88291e9c6a60dda59cf"
|
| 205 |
+
},
|
| 206 |
+
{
|
| 207 |
+
"path": "index.html",
|
| 208 |
+
"size_bytes": 4363,
|
| 209 |
+
"sha256": "83eb2f50ff918ec1c88229d0c58cde76dc5b16776d8ac33323789a6da7930c8d"
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"path": "licenses/CC-BY-4.0.txt",
|
| 213 |
+
"size_bytes": 18656,
|
| 214 |
+
"sha256": "9e5f1b3c610b9c2da5c313bf81d577a7d1acec686bdb0384edefa6df0f90cd94"
|
| 215 |
+
},
|
| 216 |
+
{
|
| 217 |
+
"path": "licenses/PBEBench-MIT.txt",
|
| 218 |
+
"size_bytes": 1068,
|
| 219 |
+
"sha256": "7ee6673ea31294494031ee577102e866bcf6438280142f822f1497cd153543e1"
|
| 220 |
+
},
|
| 221 |
+
{
|
| 222 |
+
"path": "program-structure-certificates-v1.0.0.zip",
|
| 223 |
+
"size_bytes": 27396663,
|
| 224 |
+
"sha256": "e68af75e393e34668b2d196ea2659261f428dbaf1ad8e5556a7cec1288dc4515"
|
| 225 |
+
},
|
| 226 |
+
{
|
| 227 |
+
"path": "provenance/ANALYSIS_HISTORY.json",
|
| 228 |
+
"size_bytes": 4314,
|
| 229 |
+
"sha256": "9b8872616c8a65248574431a62ddf41338651b20af31edf1eb5689d517cf55ff"
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"path": "provenance/NORMALIZATION.json",
|
| 233 |
+
"size_bytes": 1289,
|
| 234 |
+
"sha256": "10c7e5e51c9b62b4c59f76bc90c2ece4813ababcc687394757f07359279e87ce"
|
| 235 |
+
},
|
| 236 |
+
{
|
| 237 |
+
"path": "replay_fixture.py",
|
| 238 |
+
"size_bytes": 5708,
|
| 239 |
+
"sha256": "4a35106cc2c0d8e84222c037ec56b6157b489fc91d2d19fd12e5a6c686216eb8"
|
| 240 |
+
},
|
| 241 |
+
{
|
| 242 |
+
"path": "reproduce.py",
|
| 243 |
+
"size_bytes": 16662,
|
| 244 |
+
"sha256": "91998c900c54b85fd13551014abc16038de947c7741fea192309bf26edce2fb5"
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"path": "requirements.txt",
|
| 248 |
+
"size_bytes": 15,
|
| 249 |
+
"sha256": "99f43573522565d7c480efd1f954892948033915df6a4e685327e995d3396eb6"
|
| 250 |
+
},
|
| 251 |
+
{
|
| 252 |
+
"path": "results/BROADER_PANEL_READABLE.csv",
|
| 253 |
+
"size_bytes": 2275,
|
| 254 |
+
"sha256": "b2ed0a1e44acebb2c4c85e826be278a529bc9e72878cb8df6e7b2305d419bfc9"
|
| 255 |
+
},
|
| 256 |
+
{
|
| 257 |
+
"path": "results/PRIMARY_PANEL_READABLE.csv",
|
| 258 |
+
"size_bytes": 1366,
|
| 259 |
+
"sha256": "16868778301c28cdb7441572059229e3610903cd426bad122cad2eeb33c09102"
|
| 260 |
+
},
|
| 261 |
+
{
|
| 262 |
+
"path": "results/base_only_solution_counts.csv",
|
| 263 |
+
"size_bytes": 43845,
|
| 264 |
+
"sha256": "868346768fdb71fa0c80c52beee3c09ab685f14fe70f968f2c302e6bed73f144"
|
| 265 |
+
},
|
| 266 |
+
{
|
| 267 |
+
"path": "results/broader_original_immutable.json",
|
| 268 |
+
"size_bytes": 162066,
|
| 269 |
+
"sha256": "db2c1c28672538d8144e8905bde9f885065d58fd4b20c7c302218372e8b2e30e"
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"path": "results/family_paired_differences.csv",
|
| 273 |
+
"size_bytes": 322677,
|
| 274 |
+
"sha256": "10aacaa32c80bef20442cb7d1aadd97fe1cccbb2131dba0d85ca525bb787b9f3"
|
| 275 |
+
},
|
| 276 |
+
{
|
| 277 |
+
"path": "results/paired_effects.csv",
|
| 278 |
+
"size_bytes": 64158,
|
| 279 |
+
"sha256": "5085090c3147e28f00184546201d6b1d7e06b7bcc70b42a9e468625b266f4b90"
|
| 280 |
+
},
|
| 281 |
+
{
|
| 282 |
+
"path": "results/per_family_metrics.csv",
|
| 283 |
+
"size_bytes": 434378,
|
| 284 |
+
"sha256": "18e2ebd712fd2cebe6f7e766acc2d67707d29eebbf5a28ae00a0ed34aee008d6"
|
| 285 |
+
},
|
| 286 |
+
{
|
| 287 |
+
"path": "scripts/package_release.py",
|
| 288 |
+
"size_bytes": 2790,
|
| 289 |
+
"sha256": "acd2bf73ad435b0b05caf0b5973c0b74b183f5a50c0d3ea332c9a3ee492cedf3"
|
| 290 |
+
},
|
| 291 |
+
{
|
| 292 |
+
"path": "scripts/publish_space.py",
|
| 293 |
+
"size_bytes": 5419,
|
| 294 |
+
"sha256": "e0bae04da43d16062a0118f3c6aafd5b1744ebd70cada6db278288385950f5eb"
|
| 295 |
+
},
|
| 296 |
+
{
|
| 297 |
+
"path": "scripts/render_reports.py",
|
| 298 |
+
"size_bytes": 2111,
|
| 299 |
+
"sha256": "fa1a187dfedf43b0b8105aea1b6f914dfd9aba5ac04c6d3bf454146a85419bbc"
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"path": "style.css",
|
| 303 |
+
"size_bytes": 9253,
|
| 304 |
+
"sha256": "5071739230e76b4e928725ae3da566fa0a0cd5f0680b4a6b4a57d3785f13ce65"
|
| 305 |
+
},
|
| 306 |
+
{
|
| 307 |
+
"path": "verify_manifest.py",
|
| 308 |
+
"size_bytes": 1466,
|
| 309 |
+
"sha256": "ca7cd1faea2195c447c5d3596b125fd8bc93ff9a2db050403e71602c3f27ffb9"
|
| 310 |
+
},
|
| 311 |
+
{
|
| 312 |
+
"path": "worked_case.json",
|
| 313 |
+
"size_bytes": 9537,
|
| 314 |
+
"sha256": "6c80328e4c960f1b3a0d26d499cc7ef0d2e091203f48a18b6bc39988fd4851a5"
|
| 315 |
+
}
|
| 316 |
+
],
|
| 317 |
+
"result_scope": {
|
| 318 |
+
"cpu_replay": {
|
| 319 |
+
"command": "python -B reproduce.py --out replay",
|
| 320 |
+
"trajectories": 1920,
|
| 321 |
+
"arm_certificates": 960,
|
| 322 |
+
"archive_tasks": 1008,
|
| 323 |
+
"broader_reference_targets": 7680,
|
| 324 |
+
"active_targets": 3840,
|
| 325 |
+
"csv_rows": {
|
| 326 |
+
"per_family_metrics.csv": 1920,
|
| 327 |
+
"paired_effects.csv": 348,
|
| 328 |
+
"family_paired_differences.csv": 2872
|
| 329 |
+
},
|
| 330 |
+
"model_inference_calls": 0
|
| 331 |
+
},
|
| 332 |
+
"original_tables_1_2": "Original adjusted intervals retained; Qwen/Olmo base-five point estimates are recovered in the count-panel replay.",
|
| 333 |
+
"figure1_fixture": {
|
| 334 |
+
"command": "python -B replay_fixture.py",
|
| 335 |
+
"saved_outputs": 4,
|
| 336 |
+
"family_selection": "Existing Figure 1 example; both models and both conditions at base five."
|
| 337 |
+
}
|
| 338 |
+
},
|
| 339 |
+
"unavailable_records": [
|
| 340 |
+
{
|
| 341 |
+
"artifact_id": "original_discovery_trajectories",
|
| 342 |
+
"artifact_path": null,
|
| 343 |
+
"sha256": null,
|
| 344 |
+
"bytes": null,
|
| 345 |
+
"cohort": [
|
| 346 |
+
"discovery"
|
| 347 |
+
],
|
| 348 |
+
"count_base": [],
|
| 349 |
+
"models": [
|
| 350 |
+
"Qwen3-8B",
|
| 351 |
+
"Olmo-3-7B-Think"
|
| 352 |
+
],
|
| 353 |
+
"status": "not_in_package",
|
| 354 |
+
"replay_entrypoint": "none",
|
| 355 |
+
"reproduces": [],
|
| 356 |
+
"limitations": [
|
| 357 |
+
"Aggregate discovery results and readout sensitivities are documented; original trajectories are absent."
|
| 358 |
+
],
|
| 359 |
+
"documented_in": "Appendix B"
|
| 360 |
+
},
|
| 361 |
+
{
|
| 362 |
+
"artifact_id": "corrected_phi_trajectories",
|
| 363 |
+
"artifact_path": null,
|
| 364 |
+
"sha256": null,
|
| 365 |
+
"bytes": null,
|
| 366 |
+
"cohort": [
|
| 367 |
+
"original_validation"
|
| 368 |
+
],
|
| 369 |
+
"count_base": [
|
| 370 |
+
5
|
| 371 |
+
],
|
| 372 |
+
"models": [
|
| 373 |
+
"Phi-4-mini-reasoning"
|
| 374 |
+
],
|
| 375 |
+
"status": "not_in_package",
|
| 376 |
+
"replay_entrypoint": "none",
|
| 377 |
+
"reproduces": [],
|
| 378 |
+
"limitations": [
|
| 379 |
+
"Original Phi outputs are absent; manuscript aggregates and protocols remain."
|
| 380 |
+
],
|
| 381 |
+
"documented_in": "Appendices C/D/H/J"
|
| 382 |
+
},
|
| 383 |
+
{
|
| 384 |
+
"artifact_id": "attention_validation_raw_logs",
|
| 385 |
+
"artifact_path": null,
|
| 386 |
+
"sha256": null,
|
| 387 |
+
"bytes": null,
|
| 388 |
+
"cohort": [
|
| 389 |
+
"original_validation"
|
| 390 |
+
],
|
| 391 |
+
"count_base": [
|
| 392 |
+
5
|
| 393 |
+
],
|
| 394 |
+
"models": [
|
| 395 |
+
"Phi-4-mini-reasoning"
|
| 396 |
+
],
|
| 397 |
+
"status": "not_in_package",
|
| 398 |
+
"replay_entrypoint": "none",
|
| 399 |
+
"reproduces": [],
|
| 400 |
+
"limitations": [
|
| 401 |
+
"Numerical check summaries are retained; raw validation logs are absent."
|
| 402 |
+
],
|
| 403 |
+
"documented_in": "Appendix D"
|
| 404 |
+
},
|
| 405 |
+
{
|
| 406 |
+
"artifact_id": "timed_symbolic_search_records",
|
| 407 |
+
"artifact_path": null,
|
| 408 |
+
"sha256": null,
|
| 409 |
+
"bytes": null,
|
| 410 |
+
"cohort": [
|
| 411 |
+
"symbolic_search"
|
| 412 |
+
],
|
| 413 |
+
"count_base": [
|
| 414 |
+
5
|
| 415 |
+
],
|
| 416 |
+
"models": [
|
| 417 |
+
"symbolic_solver"
|
| 418 |
+
],
|
| 419 |
+
"status": "not_in_package",
|
| 420 |
+
"replay_entrypoint": "none",
|
| 421 |
+
"reproduces": [],
|
| 422 |
+
"limitations": [
|
| 423 |
+
"Search protocol and aggregate tables are retained; original timed-search records are absent."
|
| 424 |
+
],
|
| 425 |
+
"documented_in": "Section 5.4; Appendix I"
|
| 426 |
+
},
|
| 427 |
+
{
|
| 428 |
+
"artifact_id": "original_resampling_index_arrays",
|
| 429 |
+
"artifact_path": null,
|
| 430 |
+
"sha256": null,
|
| 431 |
+
"bytes": null,
|
| 432 |
+
"cohort": [
|
| 433 |
+
"original_validation"
|
| 434 |
+
],
|
| 435 |
+
"count_base": [
|
| 436 |
+
5
|
| 437 |
+
],
|
| 438 |
+
"models": [],
|
| 439 |
+
"status": "not_in_package",
|
| 440 |
+
"replay_entrypoint": "none",
|
| 441 |
+
"reproduces": [],
|
| 442 |
+
"limitations": [
|
| 443 |
+
"Original 50,000-resample index arrays are absent; reported intervals and the historical seed are documented."
|
| 444 |
+
],
|
| 445 |
+
"documented_in": "Appendix E"
|
| 446 |
+
},
|
| 447 |
+
{
|
| 448 |
+
"artifact_id": "original_three_model_prefix_record_tables",
|
| 449 |
+
"artifact_path": null,
|
| 450 |
+
"sha256": null,
|
| 451 |
+
"bytes": null,
|
| 452 |
+
"cohort": [
|
| 453 |
+
"original_validation"
|
| 454 |
+
],
|
| 455 |
+
"count_base": [
|
| 456 |
+
5
|
| 457 |
+
],
|
| 458 |
+
"models": [
|
| 459 |
+
"Qwen3-8B",
|
| 460 |
+
"Olmo-3-7B-Think",
|
| 461 |
+
"Phi-4-mini-reasoning"
|
| 462 |
+
],
|
| 463 |
+
"status": "not_in_package",
|
| 464 |
+
"replay_entrypoint": "none",
|
| 465 |
+
"reproduces": [],
|
| 466 |
+
"limitations": [
|
| 467 |
+
"Aggregate prefix results remain; Qwen/Olmo token IDs are provided, but the original three-model prefix record tables are absent."
|
| 468 |
+
],
|
| 469 |
+
"documented_in": "Appendix J"
|
| 470 |
+
}
|
| 471 |
+
],
|
| 472 |
+
"distribution": "Hugging Face static Space"
|
| 473 |
+
}
|
PUBLICATION.md
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Public edition · November 2026
|
| 2 |
+
|
| 3 |
+
**When Is Program Structure Required? Certificate-Grounded Calibration of Program-Induction Difficulty**
|
| 4 |
+
|
| 5 |
+
Wenzhang Du · arXiv:2611.24975v1 · submitted 17 November 2026.
|
| 6 |
+
|
| 7 |
+
[Paper · arXiv:2611.24975](https://arxiv.org/abs/2611.24975) · [Technical report](https://dqswordman.github.io/technical-reports/program-structure-certificates/) · [中文伴读](https://dqswordman.github.io/zh/technical-reports/program-structure-certificates/) · [Interactive Space](https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates)
|
| 8 |
+
|
| 9 |
+
The scientific artifact is tagged `v1.0.0` at `8b0f2e0dd17ee36ce9233a9053dc55c22110dfce`. The single ZIP combines that exact tagged tree with this public-edition note, the reuse notices and the English and Chinese companions. The original 49-file manifest verifies the frozen contents. `PUBLICATION_MANIFEST.json` records the source revision and checksums of the public-edition additions. The repository default branch also catalogs the live publication links.
|
| 10 |
+
|
| 11 |
+
- [Read the English companion offline](docs/report.en.md) or [read it in Chinese](docs/report.zh.md).
|
| 12 |
+
- Run `python -B verify_manifest.py` and `python -B replay_fixture.py` from the extracted folder.
|
| 13 |
+
- With Python 3.11+ and NumPy, run `python -B reproduce.py --out replay` with a new output directory. The archived inputs regenerate the three primary CSVs on CPU. [Reproduction guide](docs/reproduction.md)
|
| 14 |
+
- Cite the preprint and the v1.0.0 research artifact. [Paper](https://arxiv.org/abs/2611.24975) · [Release](https://github.com/dqswordman/program-structure-certificates/releases/tag/v1.0.0)
|
| 15 |
+
|
| 16 |
+
## Reuse
|
| 17 |
+
|
| 18 |
+
Original software by Wenzhang Du is available under the [MIT License](LICENSE). The author's original synthetic tasks, numerical measurements, companion text and original figures are available under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Credit Wenzhang Du, identify the source, link the license and indicate modifications. A local copy of the [data and documentation license](licenses/CC-BY-4.0.txt) is included.
|
| 19 |
+
|
| 20 |
+
The PBEBench files and adaptations carry their [upstream MIT notice](licenses/PBEBench-MIT.txt). Archived model-generated text, token records, model templates and other third-party material retain their applicable upstream terms. The 1,008-row retrospective roster is included as a separately identified historical audit input; its sampling and filtering chain is unresolved. The CC grant covers the author's own recorded audit measurements, with upstream rights reserved in any underlying material. The release contains no model weights. [Data provenance](DATA_PROVENANCE.md)
|
README.md
CHANGED
|
@@ -1,10 +1,25 @@
|
|
| 1 |
---
|
| 2 |
title: Program Structure Certificates
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: green
|
| 5 |
-
colorTo:
|
| 6 |
sdk: static
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
title: Program Structure Certificates
|
| 3 |
+
emoji: 🧩
|
| 4 |
colorFrom: green
|
| 5 |
+
colorTo: gray
|
| 6 |
sdk: static
|
| 7 |
+
app_file: index.html
|
| 8 |
+
license: other
|
| 9 |
+
short_description: Explore paired programs, certificates and model outcomes
|
| 10 |
+
tags:
|
| 11 |
+
- program-synthesis
|
| 12 |
+
- reproducibility
|
| 13 |
+
- evaluation
|
| 14 |
+
- arxiv:2611.24975
|
| 15 |
---
|
| 16 |
|
| 17 |
+
# When Is Program Structure Required?
|
| 18 |
+
|
| 19 |
+
Wenzhang Du · arXiv:2611.24975
|
| 20 |
+
|
| 21 |
+
Switch a single demonstration and inspect why the minimum solution grows from one rule to two. Explore eight archived model and base-count settings, with denominators and paired intervals drawn from the frozen CSVs.
|
| 22 |
+
|
| 23 |
+
[Preprint](https://arxiv.org/abs/2611.24975) · [GitHub repository](https://github.com/dqswordman/program-structure-certificates) · [English report](https://dqswordman.github.io/technical-reports/program-structure-certificates/) · [中文伴读](https://dqswordman.github.io/zh/technical-reports/program-structure-certificates/) · [Offline release](https://github.com/dqswordman/program-structure-certificates/releases/tag/v1.0.0)
|
| 24 |
+
|
| 25 |
+
The Space runs in the browser and reads saved observations. Original software uses MIT. Original task data, measurements and companion text use CC BY 4.0. See [reuse and third-party notices](https://github.com/dqswordman/program-structure-certificates/blob/main/PUBLICATION.md) and the [snapshot](./SNAPSHOT.json).
|
RELEASE_SCOPE.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
PROGRAM STRUCTURE CERTIFICATES | RESEARCH ARTIFACT v1.0.0
|
| 2 |
+
|
| 3 |
+
The archive covers 120 synthetic task families, 480 paired base-count
|
| 4 |
+
variants, 960 arm-specific certificates, 1,008 saved retrospective archive
|
| 5 |
+
tasks, 7,680 wider evaluation targets, and 1,920 normalized model outputs.
|
| 6 |
+
|
| 7 |
+
All trajectories retain generated text, generated and prompt token IDs, and
|
| 8 |
+
the deterministic selected-program status. The base-three, four, and six
|
| 9 |
+
records retain rendered prompts. The base-five records retain reconstructed
|
| 10 |
+
user messages, prompt-text hashes, and the pinned model chat templates.
|
| 11 |
+
|
| 12 |
+
The CPU entry points verify the file manifest, replay the Figure 1 example,
|
| 13 |
+
recheck the certificates and reference labels, and reproduce the count-panel
|
| 14 |
+
paired effects and family-level outcomes. The full replay uses Python 3.11+
|
| 15 |
+
and NumPy. It reads saved model outputs with zero model or GPU calls.
|
| 16 |
+
|
| 17 |
+
The paper also reports earlier discovery, corrected Phi, attention-validation,
|
| 18 |
+
timed symbolic search, and three-model prefix analyses. For these analyses,
|
| 19 |
+
the artifact inventory separates paper tables and protocols from underlying
|
| 20 |
+
historical records marked not_in_package. The original resampling arrays are
|
| 21 |
+
also listed there. The original adjusted intervals in the first two result
|
| 22 |
+
tables use their historical resampling schedule. The count-panel replay
|
| 23 |
+
recovers the Qwen and Olmo base-five point estimates.
|
| 24 |
+
|
| 25 |
+
See docs/reproduction.md for commands and expected output, docs/result-map.csv
|
| 26 |
+
for paper-to-artifact links, and DATA_PROVENANCE.md for source and reuse terms.
|
SNAPSHOT.json
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"github_commit": "f31f2ceadd54149f3f287876139f7d38dbb40b9d",
|
| 3 |
+
"github_repository": "https://github.com/dqswordman/program-structure-certificates",
|
| 4 |
+
"frozen_release": {
|
| 5 |
+
"name": "program-structure-certificates-v1.0.0.zip",
|
| 6 |
+
"bytes": 27396663,
|
| 7 |
+
"sha256": "e68af75e393e34668b2d196ea2659261f428dbaf1ad8e5556a7cec1288dc4515",
|
| 8 |
+
"scientific_commit": "8b0f2e0dd17ee36ce9233a9053dc55c22110dfce"
|
| 9 |
+
},
|
| 10 |
+
"sha256": {
|
| 11 |
+
"CITATION.cff": "76529e711e08377121c1448bf8c577e6cf25d99e4d1eb464a174a14588009b9e",
|
| 12 |
+
"DATA_PROVENANCE.md": "b0d2c322230982d5d289b12dedadc3eff4f0bf4396908ba117811b0c5923407f",
|
| 13 |
+
"GITHUB_README.md": "5d58fa73ea81a4a8bdec1892a43783d86439f95973aabfd52e6ff5dab48c3bf3",
|
| 14 |
+
"LICENSE": "020893b24fb2a5b62d2d2748f65e569e31ab7b8eeb831b5e64bb7a57e544c104",
|
| 15 |
+
"PUBLICATION.md": "2011e8cf78f537ef0f42257d0703e5ccf7927ec943b7f5153ca20f3d6470f476",
|
| 16 |
+
"README.md": "a26bbe7b397e2d5328857bb4122d6e215e372f7d9691de8121ffe631467dceb6",
|
| 17 |
+
"RELEASE_SCOPE.txt": "3701acb093ac3e9e9a348f9aa954d4a586d5c9defecc74a585aff430572ec6f1",
|
| 18 |
+
"code/canonical_readouts_v1.py": "793af7b98682cb57b5efc47b5b534e86ef90725f7c3269b4fc098d76778126a6",
|
| 19 |
+
"code/family_statistics.py": "2dd50ccb2d0e3819efa79e075573d8ce799ceb18011a9ba6229e0bc4bd095a83",
|
| 20 |
+
"code/frozen/PBEBench_functional_correctness.py": "db8ab01ef70be0c4ad34610045be4c6f6adad03977921efefc16260382901322",
|
| 21 |
+
"code/frozen/PBEBench_primitives.py": "9d68d96f6a86413f1f5849a4f589bc7ea009e5987555d7f14308bb851e92c043",
|
| 22 |
+
"code/frozen/PBEBench_utils.py": "418c27074bdcad4a3a3a1ff3a532b0ee7e9f46a2f5d41cf6da7908e5b8673b90",
|
| 23 |
+
"code/frozen/analyze_panel.py": "b6be13e25962d341bc24286c9d42b688487de49c62509598add793baa66d57aa",
|
| 24 |
+
"code/frozen/functional_score.py": "b1fffcb1ec769693cd6100423d9ba3030ce1976674e62a1c9e0cb2cf6ccee904",
|
| 25 |
+
"code/native_score.py": "3174025e5904cbd7a5f5f31385facd6a2fb7e4c64d4f857a3e1381fe792c3425",
|
| 26 |
+
"data/analysis_contract.json": "4a6991cad6350c7c6fb29c5b0a9d8f0929f3e4b0149f9c05b720970b801ee1cd",
|
| 27 |
+
"data/base_variants.jsonl": "9cd4c40b7a1468a699c40a6aacda61e73e507868d2b9cf02ad98cbbda89006d1",
|
| 28 |
+
"data/broader_inputs.jsonl": "520948e896fa37c8d0389b035ad84afc9a741a20c9e3ecf8173f49771b421fe5",
|
| 29 |
+
"data/certificate_records.jsonl": "dab7802e8005f615c3b0f3c7dac92d34ffe547b2f9b8f55e1fcd101c4de5b850",
|
| 30 |
+
"data/construction_audit.csv": "de6d2a61a2b518f8d04b0351b290a19d684c16a704d9aa8eec6cb2bf5bbd9ff9",
|
| 31 |
+
"data/families.jsonl": "c6f7814d9085f597b81d310115cf42f03efe95f7c7cdec48cacdffb69b5198b7",
|
| 32 |
+
"data/model_contract.json": "b33554033c16b56605bd384a885001d052b3a0e346487ade40b4ff9ff354466a",
|
| 33 |
+
"data/native1008.csv": "76faa6b2f0f35874a840c11e1ef6cd3180d1665a856dc64e97293492a92a0343",
|
| 34 |
+
"data/selected_programs.csv": "94ca7ea378fce2954520f6641505440257bbb1529257f36ea66fc37dd4892e67",
|
| 35 |
+
"data/synthesis_instruction.txt": "4fb7efda45f0beb0978934612eeeca36e73da3d87f4977c17dd5f4d386318f11",
|
| 36 |
+
"data/trajectories.jsonl.gz": "6d9c9af078d6ec25bb8f1e95894ed07dbd879c2e3d77b7dc5afe0081cf57ed7d",
|
| 37 |
+
"docs/asset-manifest.csv": "99eff1d2a2930b2e6ed202c23c8884b7ccf308314437701ac65b51f28b0972c0",
|
| 38 |
+
"docs/companion.json": "dbf82ff83cc84d124fa844ad26f1c406a1fb77c0dbb179a220bdc797c3b1624e",
|
| 39 |
+
"docs/figures/construction.pdf": "1224ffa9a3dfa361a7661a9fea3763250414cf84380e6cb7dc4e0c211ac9396f",
|
| 40 |
+
"docs/figures/construction.png": "73c9776e67c7fdb2524682b4a4759aa2a319843c7a667b2429cad369799291e1",
|
| 41 |
+
"docs/maintenance.md": "0cd2ddf55a1cd2694e53c65dbb5d71260038a761e9316c169ff2abe9f6ab9be4",
|
| 42 |
+
"docs/release-manifest.json": "116d011a6508cd55eb58b5668823bbd7d79501c1834754973d3d9a6ed5877604",
|
| 43 |
+
"docs/report.en.md": "24d127dbe6800d8416b5dd35a53b6487b0faebba1caadb2d6e29d81de221a4e7",
|
| 44 |
+
"docs/report.zh.md": "d944b594e45668eae6b82b63de1e6e69b1831774b6c55fa7d47d88913b2331b3",
|
| 45 |
+
"docs/reproduction.md": "fa83a6b38ffa6a677dab1ec01a0a9ba7aff394a10ff9da14f1159f2069f83bac",
|
| 46 |
+
"docs/reproduction_report.json": "4aaac315cc92a9c0b51198d8fe7285243940155f0b61913d4b8951573a47711d",
|
| 47 |
+
"docs/result-map.csv": "ac8a29e02c895deddfd10c5fdb9192c420fb8c32fb7d07f39c5f0eecd9ff2351",
|
| 48 |
+
"explore.js": "4e7a3d712bf7a7747ac7dacefe2e19e69de14afcb4ae826de163bd7703ff7e9e",
|
| 49 |
+
"explorer.json": "29f6732ad3cc85e74378578b53bfdd1558de9a485349ec27dbbd1c4078676924",
|
| 50 |
+
"fixtures/end_to_end.json": "de641ad2106e9b1fdc9e64214f0938a20e02f230ba4db88291e9c6a60dda59cf",
|
| 51 |
+
"index.html": "83eb2f50ff918ec1c88229d0c58cde76dc5b16776d8ac33323789a6da7930c8d",
|
| 52 |
+
"licenses/CC-BY-4.0.txt": "9e5f1b3c610b9c2da5c313bf81d577a7d1acec686bdb0384edefa6df0f90cd94",
|
| 53 |
+
"licenses/PBEBench-MIT.txt": "7ee6673ea31294494031ee577102e866bcf6438280142f822f1497cd153543e1",
|
| 54 |
+
"program-structure-certificates-v1.0.0.zip": "e68af75e393e34668b2d196ea2659261f428dbaf1ad8e5556a7cec1288dc4515",
|
| 55 |
+
"provenance/ANALYSIS_HISTORY.json": "9b8872616c8a65248574431a62ddf41338651b20af31edf1eb5689d517cf55ff",
|
| 56 |
+
"provenance/NORMALIZATION.json": "10c7e5e51c9b62b4c59f76bc90c2ece4813ababcc687394757f07359279e87ce",
|
| 57 |
+
"replay_fixture.py": "4a35106cc2c0d8e84222c037ec56b6157b489fc91d2d19fd12e5a6c686216eb8",
|
| 58 |
+
"reproduce.py": "91998c900c54b85fd13551014abc16038de947c7741fea192309bf26edce2fb5",
|
| 59 |
+
"requirements.txt": "99f43573522565d7c480efd1f954892948033915df6a4e685327e995d3396eb6",
|
| 60 |
+
"results/BROADER_PANEL_READABLE.csv": "b2ed0a1e44acebb2c4c85e826be278a529bc9e72878cb8df6e7b2305d419bfc9",
|
| 61 |
+
"results/PRIMARY_PANEL_READABLE.csv": "16868778301c28cdb7441572059229e3610903cd426bad122cad2eeb33c09102",
|
| 62 |
+
"results/base_only_solution_counts.csv": "868346768fdb71fa0c80c52beee3c09ab685f14fe70f968f2c302e6bed73f144",
|
| 63 |
+
"results/broader_original_immutable.json": "db2c1c28672538d8144e8905bde9f885065d58fd4b20c7c302218372e8b2e30e",
|
| 64 |
+
"results/family_paired_differences.csv": "10aacaa32c80bef20442cb7d1aadd97fe1cccbb2131dba0d85ca525bb787b9f3",
|
| 65 |
+
"results/paired_effects.csv": "5085090c3147e28f00184546201d6b1d7e06b7bcc70b42a9e468625b266f4b90",
|
| 66 |
+
"results/per_family_metrics.csv": "18e2ebd712fd2cebe6f7e766acc2d67707d29eebbf5a28ae00a0ed34aee008d6",
|
| 67 |
+
"scripts/package_release.py": "acd2bf73ad435b0b05caf0b5973c0b74b183f5a50c0d3ea332c9a3ee492cedf3",
|
| 68 |
+
"scripts/publish_space.py": "e0bae04da43d16062a0118f3c6aafd5b1744ebd70cada6db278288385950f5eb",
|
| 69 |
+
"scripts/render_reports.py": "fa1a187dfedf43b0b8105aea1b6f914dfd9aba5ac04c6d3bf454146a85419bbc",
|
| 70 |
+
"style.css": "5071739230e76b4e928725ae3da566fa0a0cd5f0680b4a6b4a57d3785f13ce65",
|
| 71 |
+
"verify_manifest.py": "ca7cd1faea2195c447c5d3596b125fd8bc93ff9a2db050403e71602c3f27ffb9",
|
| 72 |
+
"worked_case.json": "6c80328e4c960f1b3a0d26d499cc7ef0d2e091203f48a18b6bc39988fd4851a5",
|
| 73 |
+
"MANIFEST.json": "22dc510aee595af4d22ffa08fa8811969ed4bde1d678d4c76549a077e4cd46b3"
|
| 74 |
+
}
|
| 75 |
+
}
|
code/canonical_readouts_v1.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Prospective reference readouts for NECESSARY_STRUCTURE_PILOT_02.
|
| 2 |
+
Native PILOT_01 scores are immutable. This module uses no learned judge, arbitrary
|
| 3 |
+
code execution, reference program, hidden tests, or answer-dependent final selection.
|
| 4 |
+
All verification is against the finite, prompt-visible I/O specification.
|
| 5 |
+
"""
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
import ast
|
| 8 |
+
import re
|
| 9 |
+
from typing import Optional, Sequence
|
| 10 |
+
VOCAB=frozenset('abcdefghijkuvwxyz')
|
| 11 |
+
LIST_SPAN=re.compile(r'\[[^\[\]]{1,1800}\]',re.S)
|
| 12 |
+
FENCED=re.compile(r'```python\s+(.*?)```',re.S)
|
| 13 |
+
NOTATION=re.compile(r'^([abcdefghijkuvwxyz]{1,3})\s*(?:->|:|,)\s*([abcdefghijkuvwxyz]{0,3})$')
|
| 14 |
+
EOS=re.compile(r'(?:<\|endoftext\|>|<\|im_end\|>|<\|eot_id\|>)\s*$')
|
| 15 |
+
Rules=list[tuple[str,str]]
|
| 16 |
+
|
| 17 |
+
def parse_rule(rule: str)->Optional[tuple[str,str]]:
|
| 18 |
+
if not isinstance(rule,str):return None
|
| 19 |
+
m=NOTATION.fullmatch(rule.strip())
|
| 20 |
+
if m:return m.group(1),m.group(2)
|
| 21 |
+
try:
|
| 22 |
+
node=ast.parse(rule.strip(),mode='eval').body
|
| 23 |
+
if not (isinstance(node,ast.Call) and isinstance(node.func,ast.Name) and node.func.id=='replace' and len(node.args)==2 and not node.keywords):return None
|
| 24 |
+
a,b=(ast.literal_eval(x) for x in node.args)
|
| 25 |
+
if not isinstance(a,str) or not isinstance(b,str) or not 1<=len(a)<=3 or len(b)>3 or not set(a+b)<=VOCAB:return None
|
| 26 |
+
return a,b
|
| 27 |
+
except (ValueError,SyntaxError,TypeError,RecursionError):return None
|
| 28 |
+
|
| 29 |
+
def parse_list(text: str)->Optional[Rules]:
|
| 30 |
+
if len(text)>1800:return None
|
| 31 |
+
try:
|
| 32 |
+
seq=ast.literal_eval(text.strip())
|
| 33 |
+
if not isinstance(seq,list) or not 1<=len(seq)<=5 or not all(isinstance(r,str) for r in seq):return None
|
| 34 |
+
rules=[parse_rule(r) for r in seq]
|
| 35 |
+
return None if any(r is None for r in rules) else rules
|
| 36 |
+
except (ValueError,SyntaxError,TypeError,RecursionError):return None
|
| 37 |
+
|
| 38 |
+
def apply(inputs: Sequence[str],rules: Rules)->list[str]:
|
| 39 |
+
outputs=list(inputs)
|
| 40 |
+
for a,b in rules:outputs=[s.replace(a,b) for s in outputs]
|
| 41 |
+
return outputs
|
| 42 |
+
|
| 43 |
+
def final_rules(raw_text: str)->Optional[Rules]:
|
| 44 |
+
"""Select the last final-section block/list, WITHOUT consulting target outputs."""
|
| 45 |
+
if '</think>' not in raw_text:return None
|
| 46 |
+
final=EOS.sub('',raw_text.rsplit('</think>',1)[1]).strip()
|
| 47 |
+
blocks=list(FENCED.finditer(final))
|
| 48 |
+
if blocks:return parse_list(blocks[-1].group(1))
|
| 49 |
+
spans=list(LIST_SPAN.finditer(final))
|
| 50 |
+
return parse_list(spans[-1].group()) if spans else None
|
| 51 |
+
|
| 52 |
+
def readouts(raw_text: str,inputs: Sequence[str],outputs: Sequence[str])->dict:
|
| 53 |
+
if len(inputs)!=len(outputs) or not inputs:raise ValueError('Nonempty paired specification required')
|
| 54 |
+
selected=final_rules(raw_text)
|
| 55 |
+
good=[]
|
| 56 |
+
for m in LIST_SPAN.finditer(raw_text):
|
| 57 |
+
r=parse_list(m.group())
|
| 58 |
+
if r is not None and apply(inputs,r)==list(outputs):good.append({'char_end':m.end(),'rules':r})
|
| 59 |
+
return {'canonical_final_correct':int(selected is not None and apply(inputs,selected)==list(outputs)),
|
| 60 |
+
'canonical_final_parseable':selected is not None,'selected_final_rules':selected,
|
| 61 |
+
'canonical_candidate_available':bool(good),'first_verified_candidate':good[0] if good else None,
|
| 62 |
+
'candidate_count':len(good),'think_closed':'</think>' in raw_text}
|
| 63 |
+
|
| 64 |
+
def self_test()->dict:
|
| 65 |
+
tests=0
|
| 66 |
+
for text,expected in [("[\"replace('ab','c')\"]",[('ab','c')]),("['ab->c']",[('ab','c')]),("['ab,c']",[('ab','c')]),("['ab:c']",[('ab','c')]),("['a->']",[('a','')]),("[\"replace('','a')\"]",None),("[\"replace('l','a')\"]",None),("[\"replace('a','bbbb')\"]",None),("[\"replace(a='a',b='b')\"]",None),("[('a','b')]",None),("[]",None),(repr(['a->b']*6),None)]:
|
| 67 |
+
assert parse_list(text)==expected,(text,parse_list(text));tests+=1
|
| 68 |
+
xs=['ab','d'];ys=['c','d']
|
| 69 |
+
for text,final,cand in [("Thinking ['ab->c']",0,True),("Thinking ['ab->c']</think>python\n['ab->c']<|endoftext|>",1,True),("Thinking ['ab->c']</think>```python\n['ab->a']\n```",0,True),("</think>```python\n['ab->c']\n```\n```python\n['ab->a']\n```",0,True),("</think>['ab->c'] then ['ab->a']",0,True),("</think>['ab->a']",0,False)]:
|
| 70 |
+
r=readouts(text,xs,ys);assert r['canonical_final_correct']==final and r['canonical_candidate_available']==cand,(text,r);tests+=1
|
| 71 |
+
r=readouts("</think>['ab->c']",['ab','ab'],['c','b']);assert r['canonical_final_correct']==0;tests+=1
|
| 72 |
+
return {'status':'PASS','checks':tests,'role':'implementation fixtures, not scientific observations'}
|
| 73 |
+
if __name__=='__main__':
|
| 74 |
+
import json
|
| 75 |
+
print(json.dumps(self_test(),indent=2))
|
code/family_statistics.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Frozen paired family bootstrap; source archived in code/frozen/analyze_panel.py."""
|
| 2 |
+
import numpy as np
|
| 3 |
+
|
| 4 |
+
def ci(x,label,simultaneous=False):
|
| 5 |
+
x=np.asarray(x,float);n=len(x)
|
| 6 |
+
if not n:return {'n_families':0,'estimate':None,'ci_low':None,'ci_high':None}
|
| 7 |
+
# Same-family whole-row resampling. Separate conditional cohorts have separate deterministic streams.
|
| 8 |
+
rng=np.random.Generator(np.random.PCG64(20261107))
|
| 9 |
+
means=np.empty(20000)
|
| 10 |
+
for off in range(0,20000,1000):means[off:off+1000]=x[rng.integers(0,n,(1000,n))].mean(1)
|
| 11 |
+
q=[.025,.975] if not simultaneous else [.05/(2*6),1-.05/(2*6)]
|
| 12 |
+
lo,hi=np.quantile(means,q)
|
| 13 |
+
return {'n_families':n,'estimate':float(x.mean()),'ci_low':float(lo),'ci_high':float(hi),'bootstrap_replicates':20000,'interval':'percentile_family_paired' if not simultaneous else 'Bonferroni_percentile_six_model_by_newbase_comparisons'}
|
code/frozen/PBEBench_functional_correctness.py
ADDED
|
@@ -0,0 +1,525 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import re
|
| 3 |
+
import ast
|
| 4 |
+
import sys
|
| 5 |
+
import json
|
| 6 |
+
import pathlib
|
| 7 |
+
import argparse
|
| 8 |
+
import numpy as np
|
| 9 |
+
import editdistance
|
| 10 |
+
from typing import Union
|
| 11 |
+
from collections import defaultdict
|
| 12 |
+
|
| 13 |
+
module_path = str(pathlib.Path(os.path.abspath(__file__)).parent.parent.parent)
|
| 14 |
+
sys.path.append(module_path)
|
| 15 |
+
|
| 16 |
+
from src.data_generation.primitives import ProgramBFCCNode, ProgramVocabulary
|
| 17 |
+
|
| 18 |
+
def sort_dict_by_key(d: dict, reverse=False):
|
| 19 |
+
return dict(sorted(d.items(), key=lambda x: x[0], reverse=reverse))
|
| 20 |
+
|
| 21 |
+
def safe_mean(x):
|
| 22 |
+
if len(x) == 0: return 0
|
| 23 |
+
return np.mean(x)
|
| 24 |
+
|
| 25 |
+
def parse_args():
|
| 26 |
+
parser = argparse.ArgumentParser(description="Evaluate functional correctness of model generated program sequence.")
|
| 27 |
+
parser.add_argument(
|
| 28 |
+
"--model_predictions_path", type=str,
|
| 29 |
+
help="Path to the model predictions file."
|
| 30 |
+
)
|
| 31 |
+
# parser.add_argument(
|
| 32 |
+
# "--search_mode", action="store_true",
|
| 33 |
+
# help="flag to indicate model preds are a folder of search steps instead of single pred file."
|
| 34 |
+
# )
|
| 35 |
+
parser.add_argument(
|
| 36 |
+
"--vocab", type=str,
|
| 37 |
+
default="abcdefghijkuvwxyz",
|
| 38 |
+
help="vocabulary to be used for inputs/programs",
|
| 39 |
+
)
|
| 40 |
+
parser.add_argument(
|
| 41 |
+
"--max_programs", type=int, default=5,
|
| 42 |
+
help="Maximum programs the model is allowed to use.",
|
| 43 |
+
)
|
| 44 |
+
return parser.parse_args()
|
| 45 |
+
|
| 46 |
+
def classify_iod(iod: float):
|
| 47 |
+
if iod < 0.2: return "IOD < 0.2"
|
| 48 |
+
elif iod < 0.4: return "0.2 <= IOD < 0.4"
|
| 49 |
+
elif iod < 0.6: return "0.4 <= IOD < 0.6"
|
| 50 |
+
elif iod < 0.8: return "0.6 <= IOD < 0.8"
|
| 51 |
+
else: return "0.8 <= IOD"
|
| 52 |
+
|
| 53 |
+
def reward(pred_final_output: list[str], inputs: list[str], targets: list[str]) -> float:
|
| 54 |
+
pred_final_output_tok: list[str[str]] = [w.split() for w in pred_final_output]
|
| 55 |
+
inputs_tok: list[str[str]] = [w.split() for w in inputs]
|
| 56 |
+
targets_tok: list[str[str]] = [w.split() for w in targets]
|
| 57 |
+
|
| 58 |
+
pred_target_dist = sum(editdistance.eval(s, t) for s, t in zip(pred_final_output_tok, targets_tok))
|
| 59 |
+
input_target_dist = sum(editdistance.eval(s, t) for s, t in zip(inputs_tok, targets_tok))
|
| 60 |
+
|
| 61 |
+
return (1 - (pred_target_dist / input_target_dist))
|
| 62 |
+
|
| 63 |
+
def identity(x: list[str]) -> list[str]: return x
|
| 64 |
+
|
| 65 |
+
def find_degenerate_program_ids(input_dict: dict):
|
| 66 |
+
prev_outputs = input_dict["inputs"]
|
| 67 |
+
skipped_programs = [] # skip degenerate programs that don't alter any inputs.
|
| 68 |
+
for program_ind,program in enumerate(input_dict['programs']):
|
| 69 |
+
program = program.replace("\\","")
|
| 70 |
+
program_node = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary(vocab_chars))
|
| 71 |
+
next_inputs = program_node(prev_outputs)
|
| 72 |
+
# print(prev_outputs, program, next_inputs)
|
| 73 |
+
if next_inputs == prev_outputs:
|
| 74 |
+
skipped_programs.append(program_ind)
|
| 75 |
+
prev_outputs = next_inputs
|
| 76 |
+
input_dict["programs"][program_ind] = program # just to fix the issue with exccess back slashes
|
| 77 |
+
|
| 78 |
+
return skipped_programs
|
| 79 |
+
|
| 80 |
+
def extract_program(program: Union[str, ProgramBFCCNode], vocab_chars) -> ProgramBFCCNode:
|
| 81 |
+
if isinstance(program, str):
|
| 82 |
+
try:
|
| 83 |
+
program = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary(vocab_chars))
|
| 84 |
+
except IndexError as e:
|
| 85 |
+
# print(f"\x1b[31;1m{e}\x1b[0m")
|
| 86 |
+
program = identity
|
| 87 |
+
except AssertionError as e:
|
| 88 |
+
# print(f"\x1b[31;1m{e}\x1b[0m")
|
| 89 |
+
program = identity # replace invalid programs with identity programs.
|
| 90 |
+
else:
|
| 91 |
+
# assert isinstance(program, ProgramBFCCNode), f"Invalid program type: {type(program)}"
|
| 92 |
+
# print(f"\x1b[31;1mInvalid program type: {type(program)}. Replacing with identity. Check if this isn't the result of WRONG FORMATTING!!!\x1b[0m")
|
| 93 |
+
program = identity
|
| 94 |
+
|
| 95 |
+
return program
|
| 96 |
+
|
| 97 |
+
def eval_outputs(
|
| 98 |
+
inputs: list[str], outputs: list[str],
|
| 99 |
+
pred_cascade: list[Union[str, ProgramBFCCNode]],
|
| 100 |
+
vocab_chars: str="abcdefghijkuvwxyz", max_programs: int=5):
|
| 101 |
+
pred_outputs: list[str] = [i for i in inputs]
|
| 102 |
+
valid_programs = 0
|
| 103 |
+
invalid_programs = 0
|
| 104 |
+
for program in pred_cascade[:max_programs]: # we should only pick the first max_programs (m in the paper) programs predicted by the model in the cascade (in case the model disobeys the instruction and uses more programs than max_programs)
|
| 105 |
+
if isinstance(program, str):
|
| 106 |
+
try:
|
| 107 |
+
program = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary(vocab_chars))
|
| 108 |
+
valid_programs += 1
|
| 109 |
+
except IndexError as e:
|
| 110 |
+
# print(f"\x1b[31;1m{e}\x1b[0m")
|
| 111 |
+
program = identity
|
| 112 |
+
invalid_programs += 1
|
| 113 |
+
except AssertionError as e:
|
| 114 |
+
# print(f"\x1b[31;1m{e}\x1b[0m")
|
| 115 |
+
program = identity # replace invalid programs with identity programs.
|
| 116 |
+
invalid_programs += 1
|
| 117 |
+
else:
|
| 118 |
+
# assert isinstance(program, ProgramBFCCNode), f"Invalid program type: {type(program)}"
|
| 119 |
+
# print(f"\x1b[31;1mInvalid program type: {type(program)}. Replacing with identity. Check if this isn't the result of WRONG FORMATTING!!!\x1b[0m")
|
| 120 |
+
program = identity
|
| 121 |
+
invalid_programs += 1
|
| 122 |
+
pred_outputs: list[str] = program(pred_outputs)
|
| 123 |
+
|
| 124 |
+
return pred_outputs, valid_programs, invalid_programs
|
| 125 |
+
|
| 126 |
+
def eval_mean_pass_at_1(dataset, predictions: list[str], vocab_chars: str="abcdefghijkuvwxyz", max_programs: int=5):
|
| 127 |
+
# passing_programs = 0
|
| 128 |
+
passing_programs = []
|
| 129 |
+
for rec, pred_cascade in zip(dataset, predictions):
|
| 130 |
+
inputs = rec["inputs"]
|
| 131 |
+
outputs = rec["outputs"]
|
| 132 |
+
pred_outputs, valid_programs, invalid_programs = eval_outputs(inputs=inputs, outputs=outputs, pred_cascade=pred_cascade, vocab_chars=vocab_chars, max_programs=max_programs)
|
| 133 |
+
test_pass_ratio = sum([int(o == po) for o,po in zip(outputs, pred_outputs)])/len(outputs)
|
| 134 |
+
if test_pass_ratio == 1: passing_programs.append(1)
|
| 135 |
+
else: passing_programs.append(0)
|
| 136 |
+
|
| 137 |
+
return np.mean(passing_programs), passing_programs
|
| 138 |
+
|
| 139 |
+
def eval_rewards(dataset, predictions: list[str], vocab_chars: str="abcdefghijkuvwxyz", max_programs: int=5):
|
| 140 |
+
rewards = []
|
| 141 |
+
total_valid_programs = 0
|
| 142 |
+
total_invalid_programs = 0
|
| 143 |
+
for rec, pred_cascade in zip(dataset, predictions):
|
| 144 |
+
inputs = rec["inputs"]
|
| 145 |
+
outputs = rec["outputs"]
|
| 146 |
+
pred_outputs, valid_programs, invalid_programs = eval_outputs(inputs=inputs, outputs=outputs, pred_cascade=pred_cascade, vocab_chars=vocab_chars, max_programs=max_programs)
|
| 147 |
+
total_valid_programs += valid_programs
|
| 148 |
+
total_invalid_programs += invalid_programs
|
| 149 |
+
rewards.append(reward(pred_outputs, inputs, outputs))
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
return np.mean(rewards), rewards, total_valid_programs/(total_invalid_programs+total_valid_programs)
|
| 153 |
+
|
| 154 |
+
def pass_at_k(n, c, k):
|
| 155 |
+
"""
|
| 156 |
+
:param n: total number of samples
|
| 157 |
+
:param c: number of correct samples
|
| 158 |
+
:param k: k in pass@$k$
|
| 159 |
+
"""
|
| 160 |
+
if n - c < k: return 1.0
|
| 161 |
+
return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
|
| 162 |
+
|
| 163 |
+
# def eval_mean_pass_at_k(dataset, predictions: list[list[str]], k: int=5, vocab_chars: str="abcdefghijkuvwxyz", max_programs: int=5):
|
| 164 |
+
# n = len(predictions[0])
|
| 165 |
+
# assert k <= n, f"k should be less than {n}"
|
| 166 |
+
# instance_pass_at_ks = []
|
| 167 |
+
# for rec, sampled_cascades in zip(dataset, predictions):
|
| 168 |
+
# inputs = rec["inputs"]
|
| 169 |
+
# outputs = rec["outputs"]
|
| 170 |
+
# c = 0
|
| 171 |
+
# for pred_cascade in sampled_cascades:
|
| 172 |
+
# test_pass_ratio = eval_passing_test_cases(inputs=inputs, outputs=outputs, pred_cascade=pred_cascade, vocab_chars=vocab_chars, max_programs=max_programs)
|
| 173 |
+
# if test_pass_ratio > 0.999: c += 1
|
| 174 |
+
# instance_pass_at_k = pass_at_k(n, c, k)
|
| 175 |
+
# instance_pass_at_ks.append(instance_pass_at_k)
|
| 176 |
+
|
| 177 |
+
# return np.mean(instance_pass_at_ks)
|
| 178 |
+
|
| 179 |
+
def extract_first_python_block(markdown_text: str) -> str:
|
| 180 |
+
"""
|
| 181 |
+
Extracts first Python code block from a markdown string.
|
| 182 |
+
|
| 183 |
+
Args:
|
| 184 |
+
markdown_text (str): The markdown text containing code blocks.
|
| 185 |
+
|
| 186 |
+
Returns:
|
| 187 |
+
List[str]: A list of Python code blocks.
|
| 188 |
+
"""
|
| 189 |
+
code_block_pattern = r"```python\s(.*?)```"
|
| 190 |
+
matches = re.findall(code_block_pattern, markdown_text, re.DOTALL)
|
| 191 |
+
try: return [block.strip() for block in matches][0]
|
| 192 |
+
except IndexError: return "[]"
|
| 193 |
+
|
| 194 |
+
def extract_last_python_block(markdown_text: str) -> str:
|
| 195 |
+
"""
|
| 196 |
+
Extracts first Python code block from a markdown string.
|
| 197 |
+
|
| 198 |
+
Args:
|
| 199 |
+
markdown_text (str): The markdown text containing code blocks.
|
| 200 |
+
|
| 201 |
+
Returns:
|
| 202 |
+
List[str]: A list of Python code blocks.
|
| 203 |
+
"""
|
| 204 |
+
code_block_pattern = r"```python\s(.*?)```"
|
| 205 |
+
matches = re.findall(code_block_pattern, markdown_text, re.DOTALL)
|
| 206 |
+
try: return [block.strip() for block in matches][-1]
|
| 207 |
+
except IndexError: return "[]"
|
| 208 |
+
|
| 209 |
+
def extract_program_str_from_last_python_block(model_response: str) -> str:
|
| 210 |
+
try: pred = extract_last_python_block(model_response)
|
| 211 |
+
except TypeError as e:
|
| 212 |
+
if model_response is None: pass
|
| 213 |
+
else: print(e)
|
| 214 |
+
pred = "replace('x','x')"
|
| 215 |
+
|
| 216 |
+
# wrap any replace(...) that isn’t already quoted in single quotes
|
| 217 |
+
# e.g. replace('kd','ka') → 'replace('kd','ka')'
|
| 218 |
+
pred = re.sub(
|
| 219 |
+
r"(?<!['\"])(replace\([^)]*\))(?!['\"])",
|
| 220 |
+
r'"\1"',
|
| 221 |
+
pred
|
| 222 |
+
)
|
| 223 |
+
|
| 224 |
+
return pred
|
| 225 |
+
|
| 226 |
+
# def load_model_predictions(path: str):
|
| 227 |
+
# raw_data = read_jsonl(path)
|
| 228 |
+
# loaded_data = []
|
| 229 |
+
# for rec in raw_data:
|
| 230 |
+
# loaded_data.append(rec['input'])
|
| 231 |
+
# loaded_data[-1]["prediction"] = extract_first_python_block(rec['output'])
|
| 232 |
+
# print(loaded_data[-1]['prediction'])
|
| 233 |
+
|
| 234 |
+
# return loaded_data
|
| 235 |
+
|
| 236 |
+
def load_model_predictions(path: str, extract_method) -> list[dict]:
|
| 237 |
+
raw_data: list[dict] = read_jsonl(path)
|
| 238 |
+
loaded_data = []
|
| 239 |
+
for rec in raw_data:
|
| 240 |
+
# extract the code‐block text
|
| 241 |
+
if isinstance(rec['output'], dict):
|
| 242 |
+
rec['output'] = rec["output"]["text"] # fix specific for QwQ file (should correct the prediction format for future)
|
| 243 |
+
pred = extract_method(rec['output'])
|
| 244 |
+
|
| 245 |
+
# wrap any replace(...) that isn’t already quoted in single quotes
|
| 246 |
+
# e.g. replace('kd','ka') → 'replace('kd','ka')'
|
| 247 |
+
pred = re.sub(
|
| 248 |
+
r"(?<!['\"])(replace\([^)]*\))(?!['\"])",
|
| 249 |
+
r'"\1"',
|
| 250 |
+
pred
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
# assign and print
|
| 254 |
+
rec['input']["predictions"] = [pred]
|
| 255 |
+
# print(pred)
|
| 256 |
+
|
| 257 |
+
loaded_data.append(rec['input'])
|
| 258 |
+
|
| 259 |
+
return loaded_data
|
| 260 |
+
|
| 261 |
+
def load_model_predictions_multi(path: str, extract_method) -> list[dict]:
|
| 262 |
+
raw_data: list[dict] = read_jsonl(path)
|
| 263 |
+
loaded_data = []
|
| 264 |
+
null_ctr = 0
|
| 265 |
+
for rec in raw_data:
|
| 266 |
+
# extract the code‐block text
|
| 267 |
+
rec['input']["predictions"] = []
|
| 268 |
+
for output in rec['outputs']:
|
| 269 |
+
try: pred = extract_method(output)
|
| 270 |
+
except TypeError as e:
|
| 271 |
+
if output is None:
|
| 272 |
+
null_ctr += 1
|
| 273 |
+
else: print(e)
|
| 274 |
+
pred = "replace('x','x')"
|
| 275 |
+
|
| 276 |
+
# wrap any replace(...) that isn’t already quoted in single quotes
|
| 277 |
+
# e.g. replace('kd','ka') → 'replace('kd','ka')'
|
| 278 |
+
pred = re.sub(
|
| 279 |
+
r"(?<!['\"])(replace\([^)]*\))(?!['\"])",
|
| 280 |
+
r'"\1"',
|
| 281 |
+
pred
|
| 282 |
+
)
|
| 283 |
+
|
| 284 |
+
# assign and print
|
| 285 |
+
rec['input']["predictions"].append(pred)
|
| 286 |
+
# print(pred)
|
| 287 |
+
|
| 288 |
+
loaded_data.append(rec['input'])
|
| 289 |
+
print(f"found {null_ctr} nulls")
|
| 290 |
+
|
| 291 |
+
return loaded_data
|
| 292 |
+
|
| 293 |
+
def process_model_predictions_multi(raw_data: list[dict], extract_method) -> list[dict]:
|
| 294 |
+
loaded_data = []
|
| 295 |
+
null_ctr = 0
|
| 296 |
+
for rec in raw_data:
|
| 297 |
+
# extract the code‐block text
|
| 298 |
+
rec['input']["predictions"] = []
|
| 299 |
+
for output in rec['outputs']:
|
| 300 |
+
try: pred = extract_method(output)
|
| 301 |
+
except TypeError as e:
|
| 302 |
+
if output is None:
|
| 303 |
+
null_ctr += 1
|
| 304 |
+
else: print(e)
|
| 305 |
+
pred = "replace('x','x')"
|
| 306 |
+
|
| 307 |
+
# wrap any replace(...) that isn’t already quoted in single quotes
|
| 308 |
+
# e.g. replace('kd','ka') → 'replace('kd','ka')'
|
| 309 |
+
pred = re.sub(
|
| 310 |
+
r"(?<!['\"])(replace\([^)]*\))(?!['\"])",
|
| 311 |
+
r'"\1"',
|
| 312 |
+
pred
|
| 313 |
+
)
|
| 314 |
+
|
| 315 |
+
# assign and print
|
| 316 |
+
rec['input']["predictions"].append(pred)
|
| 317 |
+
# print(pred)
|
| 318 |
+
|
| 319 |
+
loaded_data.append(rec['input'])
|
| 320 |
+
print(f"found {null_ctr} nulls")
|
| 321 |
+
|
| 322 |
+
return loaded_data
|
| 323 |
+
|
| 324 |
+
def normalized_levenshtein(s1: str, s2: str) -> float:
|
| 325 |
+
distance = editdistance.eval(s1, s2)
|
| 326 |
+
max_len = max(len(s1), len(s2))
|
| 327 |
+
return distance / max_len if max_len > 0 else 0.0
|
| 328 |
+
|
| 329 |
+
def try_ast_literal_eval(string): # do some minor fixes.
|
| 330 |
+
"""Takes program sequence string and returns the extracted sequence of actual replace programs."""
|
| 331 |
+
try:
|
| 332 |
+
try:
|
| 333 |
+
program_seq = ast.literal_eval(string)
|
| 334 |
+
if isinstance(program_seq, str):
|
| 335 |
+
program_seq = [program_seq]
|
| 336 |
+
# print(program_seq, type(program_seq))
|
| 337 |
+
try:
|
| 338 |
+
assert isinstance(program_seq, list), f"program sequence of list form couldn't be extracted or derived: {program_seq}"
|
| 339 |
+
return program_seq
|
| 340 |
+
except AssertionError: return []
|
| 341 |
+
except ValueError: return []
|
| 342 |
+
except SyntaxError: return [] # empty prediction if no code blocks found.
|
| 343 |
+
|
| 344 |
+
def get_instance_complexities(model_preds, max_programs: int, vocab_chars):
|
| 345 |
+
instance_complexity = [[0 for _ in range(7)] for _ in range(len(model_preds))] # index 0-3: B,F,CB,CF counts, index 4: effective
|
| 346 |
+
pred_cascade_complexity_scores = [[] for _ in range(len(model_preds))] # score based on summing up the lengths of the alphas and betas.
|
| 347 |
+
gt_cascade_complexity_scores = [0 for _ in range(len(model_preds))] # score based on summing up the lengths of the alphas and betas.
|
| 348 |
+
|
| 349 |
+
for ind,rec in enumerate(model_preds):
|
| 350 |
+
prev_outputs = rec['inputs']
|
| 351 |
+
rec["skipped_programs"] = [] # skip degenerate programs that don't alter any inputs.
|
| 352 |
+
|
| 353 |
+
for program_ind,program in enumerate(rec['programs']):
|
| 354 |
+
program = program.replace("\\","")
|
| 355 |
+
program_node = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary(vocab_chars))
|
| 356 |
+
cascade_complexity_score = (len(program_node.predicate) + len(program_node.transform))
|
| 357 |
+
gt_cascade_complexity_scores[ind] += cascade_complexity_score
|
| 358 |
+
next_inputs = program_node(prev_outputs)
|
| 359 |
+
# print(prev_outputs, program, next_inputs)
|
| 360 |
+
if next_inputs == prev_outputs:
|
| 361 |
+
rec["skipped_programs"].append(program_ind)
|
| 362 |
+
prev_outputs = next_inputs
|
| 363 |
+
rec['programs'][program_ind] = program
|
| 364 |
+
|
| 365 |
+
for pred_ind,prediction in enumerate(rec['predictions']):
|
| 366 |
+
pred_cascade_complexity_scores[ind] = [0 for _ in range(len(rec['predictions']))]
|
| 367 |
+
pred_cascade = try_ast_literal_eval(prediction)
|
| 368 |
+
for program_ind, program in enumerate(pred_cascade):
|
| 369 |
+
try: program = program.replace("\\","")
|
| 370 |
+
except AttributeError: program = ""
|
| 371 |
+
try:
|
| 372 |
+
program_node = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%^&*()_+-=1234567890 §{}[]|\?"/?><,.'), max_window_size=20) # deliberately add a very huge vocab so we can count the true complexity.
|
| 373 |
+
cascade_complexity_score = (len(program_node.predicate) + len(program_node.transform))
|
| 374 |
+
pred_cascade_complexity_scores[ind][pred_ind] += cascade_complexity_score
|
| 375 |
+
except AssertionError:
|
| 376 |
+
pred_cascade_complexity_scores[ind][pred_ind] += 0 # invalid programs get a zero score.
|
| 377 |
+
|
| 378 |
+
# edit distance complexity feature not affected by the skipped_programs.
|
| 379 |
+
instance_complexity[ind][5] = np.mean([normalized_levenshtein(i,o) for i,o in zip(rec["inputs"], rec["outputs"])])
|
| 380 |
+
# effective cascade length after skipping degenerate programs.
|
| 381 |
+
instance_complexity[ind][4] = len(rec['programs'])-len(rec["skipped_programs"]) # effective cascade length
|
| 382 |
+
# no. of words that have changed.
|
| 383 |
+
instance_complexity[ind][6] = sum((i != o) for i,o in zip(rec['inputs'], rec['outputs']))/5
|
| 384 |
+
|
| 385 |
+
# effective relation counts after removing degenerate programs.
|
| 386 |
+
for link in rec["bfcc_dag"]:
|
| 387 |
+
if link[0] in rec["skipped_programs"] or link[-1] in rec["skipped_programs"]: continue
|
| 388 |
+
if link[1][0] == "N": pass
|
| 389 |
+
elif link[1][0] == "B" and link[0] < link[-1]: # bleeding.
|
| 390 |
+
instance_complexity[ind][0] += 1
|
| 391 |
+
elif link[1][0] == "F" and link[0] < link[-1]: # feeding.
|
| 392 |
+
instance_complexity[ind][1] += 1
|
| 393 |
+
elif link[1][0] == "B" and link[0] > link[-1]: # counter-bleeding.
|
| 394 |
+
instance_complexity[ind][2] += 1
|
| 395 |
+
elif link[1][0] == "F" and link[0] > link[-1]: # counter-feeding.
|
| 396 |
+
instance_complexity[ind][3] += 1
|
| 397 |
+
|
| 398 |
+
return instance_complexity, gt_cascade_complexity_scores, pred_cascade_complexity_scores
|
| 399 |
+
|
| 400 |
+
def pick_highest_reward_prog_seq(programs: list[str], inputs: list[str], outputs: list[str], max_programs: int=5, vocab_chars=None):
|
| 401 |
+
progs_and_rewards = []
|
| 402 |
+
for prog_seq_string in programs:
|
| 403 |
+
pred_cascade = try_ast_literal_eval(prog_seq_string)
|
| 404 |
+
pred_outputs, valid_programs, invalid_programs = eval_outputs(inputs=inputs, outputs=outputs, pred_cascade=pred_cascade, vocab_chars=vocab_chars, max_programs=max_programs)
|
| 405 |
+
progs_and_rewards.append((pred_cascade[:max_programs], reward(pred_outputs, inputs, outputs)))
|
| 406 |
+
progs_and_rewards = sorted(progs_and_rewards, reverse=True, key=lambda x: x[1]) # sort by reward.
|
| 407 |
+
# print(progs_and_rewards[0][0])
|
| 408 |
+
|
| 409 |
+
return progs_and_rewards[0][0]
|
| 410 |
+
|
| 411 |
+
def pick_highest_reward_outputs(programs: list[str], inputs: list[str], outputs: list[str], max_programs: int=5, vocab_chars=None):
|
| 412 |
+
pred_outputs_and_rewards = []
|
| 413 |
+
for prog_seq_string in programs:
|
| 414 |
+
pred_cascade = try_ast_literal_eval(prog_seq_string)
|
| 415 |
+
pred_outputs, valid_programs, invalid_programs = eval_outputs(inputs=inputs, outputs=outputs, pred_cascade=pred_cascade, vocab_chars=vocab_chars, max_programs=max_programs)
|
| 416 |
+
pred_outputs_and_rewards.append((pred_outputs, reward(pred_outputs, inputs, outputs)))
|
| 417 |
+
pred_outputs_and_rewards = sorted(pred_outputs_and_rewards, reverse=True, key=lambda x: x[1]) # sort by reward.
|
| 418 |
+
# print(progs_and_rewards[0][0])
|
| 419 |
+
|
| 420 |
+
return pred_outputs_and_rewards[0][0]
|
| 421 |
+
|
| 422 |
+
# main
|
| 423 |
+
if __name__ == "__main__":
|
| 424 |
+
from src.data_generation.utils import read_jsonl
|
| 425 |
+
|
| 426 |
+
args = parse_args()
|
| 427 |
+
|
| 428 |
+
vocab_chars = args.vocab
|
| 429 |
+
model_predictions_path = args.model_predictions_path
|
| 430 |
+
metric_values_path = os.path.join("metric_values", pathlib.Path(model_predictions_path).stem+".json")
|
| 431 |
+
|
| 432 |
+
for extraction_type,extract_method in {
|
| 433 |
+
"extract first code block": extract_first_python_block,
|
| 434 |
+
"extract last code block": extract_last_python_block,
|
| 435 |
+
}.items():
|
| 436 |
+
# try:
|
| 437 |
+
data = load_model_predictions_multi(model_predictions_path, extract_method=extract_method)
|
| 438 |
+
best_predictions = [pick_highest_reward_prog_seq(programs=rec['predictions'], inputs=rec['inputs'], outputs=rec['outputs'], max_programs=args.max_programs, vocab_chars=vocab_chars) for rec in data]
|
| 439 |
+
# print([len(p) for p in best_predictions])
|
| 440 |
+
# except KeyError:
|
| 441 |
+
# data = load_model_predictions(model_predictions_path, extract_method=extract_method)
|
| 442 |
+
# best_predictions = [try_ast_literal_eval(rec['predictions'][0]) for rec in data]
|
| 443 |
+
# [try_ast_literal_eval(rec['prediction']) for rec in data]
|
| 444 |
+
# exit()
|
| 445 |
+
instance_complexities, gt_cascade_complexity_scores, pred_cascade_complexity_scores = get_instance_complexities(data, max_programs=args.max_programs, vocab_chars=vocab_chars)
|
| 446 |
+
# print(instance_complexities[0])
|
| 447 |
+
|
| 448 |
+
mean_pass_at_1, inst_level_passing_progs = eval_mean_pass_at_1(data, best_predictions, vocab_chars=vocab_chars, max_programs=args.max_programs)
|
| 449 |
+
reward_at_1, inst_level_rewards, valid_program_rate = eval_rewards(data, best_predictions, vocab_chars=vocab_chars, max_programs=args.max_programs)
|
| 450 |
+
|
| 451 |
+
metric_values = []
|
| 452 |
+
for passing, reward_val in zip(inst_level_passing_progs, inst_level_rewards):
|
| 453 |
+
if passing == 1: assert reward_val == 1, f"reward: {reward_val}, passing: {passing}"
|
| 454 |
+
if reward_val < 1: assert passing == 0
|
| 455 |
+
metric_values.append({"passing": passing, "reward": reward_val})
|
| 456 |
+
with open(metric_values_path, "w") as f:
|
| 457 |
+
json.dump(metric_values, f, indent=4)
|
| 458 |
+
|
| 459 |
+
# sampled_predictions = [[ast.literal_eval(pred) for pred in rec['prediction']] for rec in data]
|
| 460 |
+
# pass_at_1 = eval_mean_pass_at_k(data, sampled_predictions, k=1)
|
| 461 |
+
# pass_at_3 = eval_mean_pass_at_k(data, sampled_predictions, k=3)
|
| 462 |
+
# pass_at_5 = eval_mean_pass_at_k(data, sampled_predictions, k=5)
|
| 463 |
+
# pass_at_10 = eval_mean_pass_at_k(data, sampled_predictions, k=10)
|
| 464 |
+
|
| 465 |
+
print(extraction_type)
|
| 466 |
+
print(f"pass@1: {mean_pass_at_1:.4f}")
|
| 467 |
+
print(f"reward@1: {reward_at_1:.4f}")
|
| 468 |
+
print(f"valid program rate: {valid_program_rate:.4f}")
|
| 469 |
+
print(f"gt complexity: {np.mean(gt_cascade_complexity_scores):.2f}")
|
| 470 |
+
print(f"pred complexity: {np.mean([safe_mean(p) for p in pred_cascade_complexity_scores]):.2f}")
|
| 471 |
+
|
| 472 |
+
perf_by_cascade_length = defaultdict(lambda: [])
|
| 473 |
+
perf_by_num_relns = defaultdict(lambda: [])
|
| 474 |
+
perf_by_num_uniq_relns = defaultdict(lambda: [])
|
| 475 |
+
perf_by_io_dist = defaultdict(lambda: [])
|
| 476 |
+
|
| 477 |
+
cascade_length_dist = defaultdict(lambda: 0)
|
| 478 |
+
num_relns_dist = defaultdict(lambda: 0)
|
| 479 |
+
io_dist = defaultdict(lambda: 0)
|
| 480 |
+
|
| 481 |
+
for i in range(len(data)):
|
| 482 |
+
effective_cascade_length = instance_complexities[i][4]
|
| 483 |
+
num_relns = sum(instance_complexities[i][:4])
|
| 484 |
+
num_uniq_relns = sum([int(rel_count > 0) for rel_count in instance_complexities[i][:4]])
|
| 485 |
+
io_distance = instance_complexities[i][5]
|
| 486 |
+
|
| 487 |
+
perf_by_cascade_length[effective_cascade_length].append(inst_level_passing_progs[i])
|
| 488 |
+
perf_by_num_relns[num_relns].append(inst_level_passing_progs[i])
|
| 489 |
+
perf_by_num_uniq_relns[num_uniq_relns].append(inst_level_passing_progs[i])
|
| 490 |
+
perf_by_io_dist[classify_iod(io_distance)].append(inst_level_passing_progs[i])
|
| 491 |
+
# assert effective_cascade_length == instance_complexities[i][4]
|
| 492 |
+
cascade_length_dist[effective_cascade_length] += 1
|
| 493 |
+
num_relns_dist[num_relns] += 1
|
| 494 |
+
io_dist[classify_iod(io_distance)] += 1
|
| 495 |
+
|
| 496 |
+
perf_by_cascade_length = dict(perf_by_cascade_length)
|
| 497 |
+
for k in perf_by_cascade_length:
|
| 498 |
+
perf_by_cascade_length[k] = (round(sum(perf_by_cascade_length[k])/len(perf_by_cascade_length[k]),4),len(perf_by_cascade_length[k]))
|
| 499 |
+
for k in perf_by_num_relns:
|
| 500 |
+
perf_by_num_relns[k] = (round(sum(perf_by_num_relns[k])/len(perf_by_num_relns[k]),4),len(perf_by_num_relns[k]))
|
| 501 |
+
for k in perf_by_io_dist:
|
| 502 |
+
perf_by_io_dist[k] = (round(sum(perf_by_io_dist[k])/len(perf_by_io_dist[k]),4),len(perf_by_io_dist[k]))
|
| 503 |
+
for k in perf_by_num_uniq_relns:
|
| 504 |
+
perf_by_num_uniq_relns[k] = (round(sum(perf_by_num_uniq_relns[k])/len(perf_by_num_uniq_relns[k]),4),len(perf_by_num_uniq_relns[k]))
|
| 505 |
+
|
| 506 |
+
cascade_length_dist = dict(cascade_length_dist)
|
| 507 |
+
num_relns_dist = dict(num_relns_dist)
|
| 508 |
+
io_dist = dict(io_dist)
|
| 509 |
+
|
| 510 |
+
perf_by_cascade_length = dict(perf_by_cascade_length)
|
| 511 |
+
perf_by_num_relns = dict(perf_by_num_relns)
|
| 512 |
+
perf_by_io_dist = dict(perf_by_io_dist)
|
| 513 |
+
|
| 514 |
+
print("Perf. by Cascade Length:", sort_dict_by_key(perf_by_cascade_length))
|
| 515 |
+
print("Perf. by # Relations:", sort_dict_by_key(perf_by_num_relns))
|
| 516 |
+
print("Perf. by # Unique Relations (e.g 1101 category = 3):", sort_dict_by_key(perf_by_num_uniq_relns))
|
| 517 |
+
print("Perf. by IOD:", {k: perf_by_io_dist.get(k) for k in ['IOD < 0.2', '0.2 <= IOD < 0.4', '0.4 <= IOD < 0.6', '0.6 <= IOD < 0.8']})
|
| 518 |
+
|
| 519 |
+
# print(cascade_length_dist)
|
| 520 |
+
# print(num_relns_dist)
|
| 521 |
+
# print(io_dist)
|
| 522 |
+
|
| 523 |
+
# print("pass@3:", pass_at_3)
|
| 524 |
+
# print("pass@5:", pass_at_5)
|
| 525 |
+
# print("pass@10:", pass_at_10)
|
code/frozen/PBEBench_primitives.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
import json
|
| 4 |
+
import string
|
| 5 |
+
import pathlib
|
| 6 |
+
from enum import Enum
|
| 7 |
+
from tqdm import tqdm
|
| 8 |
+
|
| 9 |
+
module_path = str(pathlib.Path(os.path.abspath(__file__)).parent.parent.parent)
|
| 10 |
+
sys.path.append(module_path)
|
| 11 |
+
|
| 12 |
+
from src.data_generation.utils import read_jsonl
|
| 13 |
+
|
| 14 |
+
class ProgramVocabulary(set):
|
| 15 |
+
"""The set of characters used for creating predicate (input) and
|
| 16 |
+
transform (output) windows where a given program operates
|
| 17 |
+
"""
|
| 18 |
+
def __init__(self, *args, **kwargs):
|
| 19 |
+
if len(args) > 2: # just some syntax sugar.
|
| 20 |
+
super().__init__(list(args))
|
| 21 |
+
else: super().__init__(*args, **kwargs)
|
| 22 |
+
|
| 23 |
+
def __str__(self):
|
| 24 |
+
return "".join(list(self))
|
| 25 |
+
|
| 26 |
+
def __repr__(self):
|
| 27 |
+
return "".join(list(self))
|
| 28 |
+
|
| 29 |
+
def has_word(self, word: str):
|
| 30 |
+
word_vocab = set(list(word.strip()))
|
| 31 |
+
|
| 32 |
+
return word_vocab.issubset(self)
|
| 33 |
+
|
| 34 |
+
# Possible edges/relations between programs.
|
| 35 |
+
class ProgramBFCCEdge(Enum):
|
| 36 |
+
N = "N" #"Neutral"
|
| 37 |
+
F = "F" #'Feeding'
|
| 38 |
+
B = "B" #'Bleeding'
|
| 39 |
+
|
| 40 |
+
class ProgramBFCCNode:
|
| 41 |
+
"""Abstract representation of program in the BFCC interaction graph."""
|
| 42 |
+
def __init__(self, predicate: str, transform: str, vocabulary: ProgramVocabulary=ProgramVocabulary('a','b','c','d','e','f'), max_window_size: int=3):
|
| 43 |
+
self.predicate = predicate
|
| 44 |
+
self.transform = transform
|
| 45 |
+
self.vocabulary = vocabulary
|
| 46 |
+
self.max_window_size = max_window_size
|
| 47 |
+
|
| 48 |
+
@property
|
| 49 |
+
def alpha(self):
|
| 50 |
+
return self.predicate
|
| 51 |
+
|
| 52 |
+
@property
|
| 53 |
+
def beta(self):
|
| 54 |
+
return self.transform
|
| 55 |
+
|
| 56 |
+
@classmethod
|
| 57 |
+
def from_string(cls, string_repr: str, max_window_size: int=3, vocabulary: ProgramVocabulary=ProgramVocabulary('a','b','c','d','e','f')):
|
| 58 |
+
self = cls("","")
|
| 59 |
+
self.vocabulary = vocabulary
|
| 60 |
+
self.max_window_size = max_window_size
|
| 61 |
+
assert string_repr.strip() != '', "passed blank program input"
|
| 62 |
+
assert 'replace(' in string_repr, f"replace function not found in program string: {string_repr}"
|
| 63 |
+
|
| 64 |
+
# extract predicate and transform
|
| 65 |
+
predicate = string_repr.split("replace(")[-1].split(",")[0].strip()
|
| 66 |
+
transform = string_repr.split("replace(")[-1].split(",")[1].split(")")[0].strip()
|
| 67 |
+
|
| 68 |
+
# strip quotations off
|
| 69 |
+
predicate = predicate.replace("'","").replace('"','')
|
| 70 |
+
transform = transform.replace("'","").replace('"','')
|
| 71 |
+
# print("predicate:", predicate)
|
| 72 |
+
# print("transform:", transform)
|
| 73 |
+
|
| 74 |
+
# restrict the length of predicate and transform windows (no. of characters in them).
|
| 75 |
+
assert len(predicate) <= self.max_window_size, f"predicate has more than {self.max_window_size} characters: {predicate} in ({string_repr})"
|
| 76 |
+
assert len(transform) <= self.max_window_size, f"transform has more than {self.max_window_size} characters: {transform} in ({string_repr})"
|
| 77 |
+
|
| 78 |
+
# restrict the vocabulary of predicates and transforms.
|
| 79 |
+
assert self.vocabulary.has_word(predicate), f"the predicate {predicate} has some symbols not in the vocabulary: {self.vocabulary}"
|
| 80 |
+
assert self.vocabulary.has_word(transform), f"the transform {transform} has some symbols not in the vocabulary: {self.vocabulary}"
|
| 81 |
+
|
| 82 |
+
self.predicate = predicate
|
| 83 |
+
self.transform = transform
|
| 84 |
+
|
| 85 |
+
return self
|
| 86 |
+
|
| 87 |
+
def __call__(self, inputs: list[str]) -> list[str]:
|
| 88 |
+
outputs: list[str] = []
|
| 89 |
+
for input in inputs:
|
| 90 |
+
output = input.replace(self.predicate, self.transform)
|
| 91 |
+
outputs.append(output)
|
| 92 |
+
|
| 93 |
+
return outputs
|
| 94 |
+
|
| 95 |
+
def __eq__(self, other):
|
| 96 |
+
return self.predicate == other.predicate and self.transform == other.transform
|
| 97 |
+
|
| 98 |
+
def __repr__(self):
|
| 99 |
+
return f'replace("{self.predicate}", "{self.transform}")'
|
| 100 |
+
|
| 101 |
+
def __str__(self):
|
| 102 |
+
return self.__repr__()
|
| 103 |
+
|
| 104 |
+
class ProgramBFCCInteractionDAG:
|
| 105 |
+
def __init__(self, triples):
|
| 106 |
+
self.triples
|
| 107 |
+
|
| 108 |
+
# main
|
| 109 |
+
if __name__ == "__main__":
|
| 110 |
+
data = read_jsonl('data/example_data.jsonl')
|
| 111 |
+
# validating data.
|
| 112 |
+
for instance in data:
|
| 113 |
+
for program in instance["programs"]:
|
| 114 |
+
# check if a valid program can be constructed from string outputs.
|
| 115 |
+
program_node = ProgramBFCCNode.from_string(program, vocabulary="abcdefghijkuvwxyz")
|
| 116 |
+
print(program_node.predicate)
|
| 117 |
+
print(program_node.transform)
|
| 118 |
+
# TODO: add checks for BFCC DAG.
|
code/frozen/PBEBench_utils.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from tqdm import tqdm
|
| 3 |
+
|
| 4 |
+
def read_jsonl(path: str):
|
| 5 |
+
data = []
|
| 6 |
+
with open(path) as f:
|
| 7 |
+
for line in f:
|
| 8 |
+
line = line.strip()
|
| 9 |
+
data.append(json.loads(line))
|
| 10 |
+
|
| 11 |
+
return data
|
code/frozen/analyze_panel.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pre-output analysis specification: family pairing, fixed endpoints, no outcome selection."""
|
| 2 |
+
from rb_common import *
|
| 3 |
+
import collections,time
|
| 4 |
+
import numpy as np
|
| 5 |
+
from pilot_common import apply_pairs
|
| 6 |
+
import canonical_readouts_v1 as c
|
| 7 |
+
import legacy_readouts as legacy
|
| 8 |
+
|
| 9 |
+
ENDPOINTS=['displayed_success','short_solution_delivered','common9_consistency','unshown_pair_success','original_unseen9_success','uniform_fraction','active_fraction','candidate_available','native_success']
|
| 10 |
+
def ci(x,label,simultaneous=False):
|
| 11 |
+
x=np.asarray(x,float);n=len(x)
|
| 12 |
+
if not n:return {'n_families':0,'estimate':None,'ci_low':None,'ci_high':None}
|
| 13 |
+
# Same-family whole-row resampling. Separate conditional cohorts have separate deterministic streams.
|
| 14 |
+
rng=np.random.Generator(np.random.PCG64(20261107))
|
| 15 |
+
means=np.empty(20000)
|
| 16 |
+
for off in range(0,20000,1000):means[off:off+1000]=x[rng.integers(0,n,(1000,n))].mean(1)
|
| 17 |
+
q=[.025,.975] if not simultaneous else [.05/(2*6),1-.05/(2*6)]
|
| 18 |
+
lo,hi=np.quantile(means,q)
|
| 19 |
+
return {'n_families':n,'estimate':float(x.mean()),'ci_low':float(lo),'ci_high':float(hi),'bootstrap_replicates':20000,'interval':'percentile_family_paired' if not simultaneous else 'Bonferroni_percentile_six_model_by_newbase_comparisons'}
|
| 20 |
+
def main():
|
| 21 |
+
started=time.perf_counter();frozen=assert_freeze()
|
| 22 |
+
families={f['sample_id']:f for f in rows(OLD/'artifacts/families.jsonl')};variants=rows(RUN/'base_variant_manifest.jsonl');vby={(v['sample_id'],v['base_count']):v for v in variants}
|
| 23 |
+
audits=collections.defaultdict(dict)
|
| 24 |
+
for r in rows(A/'E1/audit_inputs.jsonl'):audits[r['sample_id']][r['stratum']]=r['accepted']
|
| 25 |
+
selected={(r['model_id'],r['sample_id'],r['structure']):r for r in rows(OLD/'artifacts/selected_finals.jsonl') if r['model_id'] in ['M1','M2']}
|
| 26 |
+
oldtexts={}
|
| 27 |
+
for mid in ['M1','M2']:
|
| 28 |
+
for path in sorted((OLD/f'artifacts/trajectories/{mid}').glob('batch_*.json')):
|
| 29 |
+
for t in read(path)['trajectories']:oldtexts[mid,t['sample_id'],t['variant']]=t
|
| 30 |
+
records=[]
|
| 31 |
+
for mid in ['M1','M2']:
|
| 32 |
+
for v in variants:
|
| 33 |
+
if v['certificate_status']=='INVALID_OR_UNRESOLVED':continue
|
| 34 |
+
sid=v['sample_id'];b=v['base_count'];f=families[sid];w=f['witness_pairs'][f['displayed_witness_index']-1];other=f['witness_pairs'][2-f['displayed_witness_index']]
|
| 35 |
+
for arm in ['CONTROL','SEPARATION']:
|
| 36 |
+
request=f'{mid}|{sid}|B{b}|{arm}'
|
| 37 |
+
if b==5:
|
| 38 |
+
t=oldtexts[mid,sid,arm];s=selected[mid,sid,arm]['selected_final'];origin='CACHED_BASE5'
|
| 39 |
+
else:
|
| 40 |
+
path=RUN/'raw'/mid/(request.replace('|','_')+'.json')
|
| 41 |
+
if not path.exists():continue
|
| 42 |
+
t=read(path);s=t['selected_final'];origin='NEW_R1'
|
| 43 |
+
rules=s['rules'];spec=v['arms'][arm];xs=spec['inputs'];ys=spec['outputs']
|
| 44 |
+
def ok(xx,yy):return int(rules is not None and c.apply(xx,rules)==yy)
|
| 45 |
+
allx=f['original_inputs']+[p[k]['input'] for p in f['witness_pairs'] for k in ['control','separation']]
|
| 46 |
+
ally=f['original_outputs']+[p[k]['output'] for p in f['witness_pairs'] for k in ['control','separation']]
|
| 47 |
+
hidden=[(x,y) for x,y in zip(allx,ally) if x not in f['original_inputs'] and x!=w[arm.lower()]['input']]
|
| 48 |
+
# Deleted original base examples do not become new hidden tests.
|
| 49 |
+
assert len(hidden)==3 and not set(x for x,y in hidden)&set(xs)
|
| 50 |
+
ux=[other[k]['input'] for k in ['control','separation']];uy=[other[k]['output'] for k in ['control','separation']]
|
| 51 |
+
metrics={'displayed_success':ok(xs,ys),'short_solution_delivered':int(ok(xs,ys) and len(rules)==1),'common9_consistency':ok(allx,ally),'unshown_pair_success':ok(ux,uy),'original_unseen9_success':ok([x for x,y in hidden],[y for x,y in hidden])}
|
| 52 |
+
for st,name in [('UNIFORM','uniform_fraction'),('ACTIVE','active_fraction')]:
|
| 53 |
+
aa=audits[sid][st];assert len(aa)==32 and not set(x['input'] for x in aa)&set(xs)
|
| 54 |
+
metrics[name]=float(np.mean([rules is not None and c.apply([x['input']],rules)==[x['output']] for x in aa]))
|
| 55 |
+
candidates=[c.parse_list(m.group()) for m in c.LIST_SPAN.finditer(t['raw_text'])]
|
| 56 |
+
metrics['candidate_available']=int(any(r is not None and c.apply(xs,r)==ys for r in candidates))
|
| 57 |
+
metrics['native_success']=int(legacy.score(t['raw_text'],xs,ys)['pass_at_1'])
|
| 58 |
+
records.append({'request_id':request,'sample_id':sid,'model_id':mid,'base_count':b,'arm':arm,'certificate_status':v['certificate_status'],'origin':origin,'parseable':s['parseable'],'selected_rule_count':len(rules) if rules is not None else None,'generated_tokens':t['generated_tokens'],'stop_reason':t['stop_reason'],'scientific_invalid_final':not s['parseable'],**metrics})
|
| 59 |
+
csvwrite(RUN/'per_family_metrics.csv',records)
|
| 60 |
+
rby={(r['model_id'],r['sample_id'],r['base_count'],r['arm']):r for r in records}
|
| 61 |
+
common=sorted(sid for sid in families if all(vby[sid,b]['certificate_status']=='CERT_1_TO_2' for b in [3,4,5,6]))
|
| 62 |
+
effects=[];family_deltas=[]
|
| 63 |
+
for mid in ['M1','M2']:
|
| 64 |
+
for b in [3,4,5,6]:
|
| 65 |
+
for group in ['CERT_1_TO_2','ALL_LEGAL','SHORTCUT_RETURN_1_TO_1','COMMON_CERT_INTERSECTION']:
|
| 66 |
+
eligible=sorted(sid for sid in families if vby[sid,b]['certificate_status']!='INVALID_OR_UNRESOLVED' and (group=='ALL_LEGAL' or group=='COMMON_CERT_INTERSECTION' and sid in common or group==vby[sid,b]['certificate_status']))
|
| 67 |
+
present=[sid for sid in eligible if all((mid,sid,b,a) in rby for a in ['CONTROL','SEPARATION'])]
|
| 68 |
+
for metric in ENDPOINTS:
|
| 69 |
+
ctrl=np.array([rby[mid,sid,b,'CONTROL'][metric] for sid in present],float);sep=np.array([rby[mid,sid,b,'SEPARATION'][metric] for sid in present],float);delta=sep-ctrl
|
| 70 |
+
result=ci(delta,f'{mid}|{b}|{group}|{metric}')
|
| 71 |
+
effects.append({'model_id':mid,'base_count':b,'cohort':group,'metric':metric,'contrast':'SEPARATION_MINUS_CONTROL','planned_n':len(eligible),'completed_n':len(present),'control_numerator_sum':float(ctrl.sum()),'separation_numerator_sum':float(sep.sum()),'control_mean':float(ctrl.mean()) if len(ctrl) else None,'separation_mean':float(sep.mean()) if len(sep) else None,**result})
|
| 72 |
+
if group=='COMMON_CERT_INTERSECTION' and b!=5:
|
| 73 |
+
full=[sid for sid in present if all((mid,sid,5,a) in rby for a in ['CONTROL','SEPARATION'])]
|
| 74 |
+
dd=np.array([(rby[mid,sid,b,'SEPARATION'][metric]-rby[mid,sid,b,'CONTROL'][metric])-(rby[mid,sid,5,'SEPARATION'][metric]-rby[mid,sid,5,'CONTROL'][metric]) for sid in full])
|
| 75 |
+
effects.append({'model_id':mid,'base_count':b,'cohort':group,'metric':metric,'contrast':'DELTA_BASE_MINUS_DELTA_BASE5',**ci(dd,f'{mid}|{b}|DID|{metric}')})
|
| 76 |
+
if metric=='displayed_success':effects.append({'model_id':mid,'base_count':b,'cohort':group,'metric':metric,'contrast':'DELTA_BASE_MINUS_DELTA_BASE5_SIMULTANEOUS',**ci(dd,f'{mid}|{b}|DID|{metric}',True)})
|
| 77 |
+
for sid in present:
|
| 78 |
+
family_deltas.append({'model_id':mid,'sample_id':sid,'base_count':b,'cohort':group,**{m:rby[mid,sid,b,'SEPARATION'][m]-rby[mid,sid,b,'CONTROL'][m] for m in ENDPOINTS}})
|
| 79 |
+
csvwrite(RUN/'paired_effects.csv',effects);csvwrite(RUN/'results/family_paired_differences.csv',family_deltas)
|
| 80 |
+
oldbroader=read(A/'E1/extra_audit_results.json');write(RUN/'results/broader_original_immutable.json',oldbroader)
|
| 81 |
+
write(RUN/'results/analysis_completeness.json',{'updated_utc':now(),'records_completed':len(records),'records_planned':1920,'new_completed':sum(r['origin']=='NEW_R1' for r in records),'base5_reused':sum(r['origin']=='CACHED_BASE5' for r in records),'common_cert_intersection_n':len(common),'common_cert_family_ids':common,'scientific_invalid_outputs':sum(r['scientific_invalid_final'] for r in records),'wall_seconds':time.perf_counter()-started,'complete':len(records)==1920,'independent_unit':'family','bootstrap_seed':20261107,'bootstrap_actual_stream':'PCG64(20261107); same sorted-family resampling stream reused jointly across endpoints/model/base with the same cohort; paired rows and differences resampled together','status':'COMPLETED' if len(records)==1920 else 'PARTIAL','common9_scope':'Original nine-example finite consistency includes original visible examples; original_unseen9 excludes originally shown examples and never promotes removed bases to new tests.'})
|
| 82 |
+
print('ANALYSIS',len(records),'rows',len(effects),'effects','commonN',len(common),flush=True)
|
| 83 |
+
if __name__=='__main__':main()
|
code/frozen/functional_score.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Frozen native scoring path; the sole alignment is empty-alpha identity."""
|
| 2 |
+
import re,types
|
| 3 |
+
from .common import VOCAB
|
| 4 |
+
from src.metrics import functional_correctness as native
|
| 5 |
+
from src.data_generation.primitives import ProgramBFCCNode
|
| 6 |
+
|
| 7 |
+
class PaperConstrainedNode(ProgramBFCCNode):
|
| 8 |
+
@classmethod
|
| 9 |
+
def from_string(cls,*args,**kwargs):
|
| 10 |
+
node=super().from_string(*args,**kwargs)
|
| 11 |
+
assert len(node.alpha)>0, 'paper forbids empty alpha'
|
| 12 |
+
return node
|
| 13 |
+
|
| 14 |
+
_globals=dict(native.eval_outputs.__globals__)
|
| 15 |
+
_globals['ProgramBFCCNode']=PaperConstrainedNode
|
| 16 |
+
eval_outputs=types.FunctionType(native.eval_outputs.__code__,_globals,'paper_eval_outputs',native.eval_outputs.__defaults__)
|
| 17 |
+
def extract(text):
|
| 18 |
+
return native.try_ast_literal_eval(native.extract_program_str_from_last_python_block(text))
|
| 19 |
+
def score(text,inputs,outputs):
|
| 20 |
+
cascade=extract(text)
|
| 21 |
+
predicted,valid,invalid=eval_outputs(inputs,outputs,cascade,VOCAB,5)
|
| 22 |
+
reasons=[]
|
| 23 |
+
for item in cascade[:5]:
|
| 24 |
+
if not isinstance(item,str): reasons.append('NON_STRING_RULE'); continue
|
| 25 |
+
try:
|
| 26 |
+
node=PaperConstrainedNode.from_string(item,vocabulary=native.ProgramVocabulary(VOCAB))
|
| 27 |
+
except (AssertionError,IndexError) as exc:
|
| 28 |
+
reasons.append(str(exc) or type(exc).__name__)
|
| 29 |
+
return {'pass_at_1':int(predicted==outputs),'predicted_outputs':predicted,'predicted_cascade':cascade,'valid_rules':valid,'invalid_rules':invalid,'invalid_rule_reasons':reasons,'complete_python_block_found':bool(re.search(r'```python\s(.*?)```',text,re.S)),'empty_cascade':not cascade}
|
code/native_score.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Portable native scorer: verbatim used definitions from the archived scientific sources.
|
| 2 |
+
The source files are retained in code/frozen. Only imports/namespace binding are adapted.
|
| 3 |
+
No generated program is executed as Python code.
|
| 4 |
+
"""
|
| 5 |
+
import ast, re, types
|
| 6 |
+
from typing import Union
|
| 7 |
+
VOCAB="abcdefghijkuvwxyz"
|
| 8 |
+
|
| 9 |
+
class ProgramVocabulary(set):
|
| 10 |
+
"""The set of characters used for creating predicate (input) and
|
| 11 |
+
transform (output) windows where a given program operates
|
| 12 |
+
"""
|
| 13 |
+
def __init__(self, *args, **kwargs):
|
| 14 |
+
if len(args) > 2: # just some syntax sugar.
|
| 15 |
+
super().__init__(list(args))
|
| 16 |
+
else: super().__init__(*args, **kwargs)
|
| 17 |
+
|
| 18 |
+
def __str__(self):
|
| 19 |
+
return "".join(list(self))
|
| 20 |
+
|
| 21 |
+
def __repr__(self):
|
| 22 |
+
return "".join(list(self))
|
| 23 |
+
|
| 24 |
+
def has_word(self, word: str):
|
| 25 |
+
word_vocab = set(list(word.strip()))
|
| 26 |
+
|
| 27 |
+
return word_vocab.issubset(self)
|
| 28 |
+
|
| 29 |
+
class ProgramBFCCNode:
|
| 30 |
+
"""Abstract representation of program in the BFCC interaction graph."""
|
| 31 |
+
def __init__(self, predicate: str, transform: str, vocabulary: ProgramVocabulary=ProgramVocabulary('a','b','c','d','e','f'), max_window_size: int=3):
|
| 32 |
+
self.predicate = predicate
|
| 33 |
+
self.transform = transform
|
| 34 |
+
self.vocabulary = vocabulary
|
| 35 |
+
self.max_window_size = max_window_size
|
| 36 |
+
|
| 37 |
+
@property
|
| 38 |
+
def alpha(self):
|
| 39 |
+
return self.predicate
|
| 40 |
+
|
| 41 |
+
@property
|
| 42 |
+
def beta(self):
|
| 43 |
+
return self.transform
|
| 44 |
+
|
| 45 |
+
@classmethod
|
| 46 |
+
def from_string(cls, string_repr: str, max_window_size: int=3, vocabulary: ProgramVocabulary=ProgramVocabulary('a','b','c','d','e','f')):
|
| 47 |
+
self = cls("","")
|
| 48 |
+
self.vocabulary = vocabulary
|
| 49 |
+
self.max_window_size = max_window_size
|
| 50 |
+
assert string_repr.strip() != '', "passed blank program input"
|
| 51 |
+
assert 'replace(' in string_repr, f"replace function not found in program string: {string_repr}"
|
| 52 |
+
|
| 53 |
+
# extract predicate and transform
|
| 54 |
+
predicate = string_repr.split("replace(")[-1].split(",")[0].strip()
|
| 55 |
+
transform = string_repr.split("replace(")[-1].split(",")[1].split(")")[0].strip()
|
| 56 |
+
|
| 57 |
+
# strip quotations off
|
| 58 |
+
predicate = predicate.replace("'","").replace('"','')
|
| 59 |
+
transform = transform.replace("'","").replace('"','')
|
| 60 |
+
# print("predicate:", predicate)
|
| 61 |
+
# print("transform:", transform)
|
| 62 |
+
|
| 63 |
+
# restrict the length of predicate and transform windows (no. of characters in them).
|
| 64 |
+
assert len(predicate) <= self.max_window_size, f"predicate has more than {self.max_window_size} characters: {predicate} in ({string_repr})"
|
| 65 |
+
assert len(transform) <= self.max_window_size, f"transform has more than {self.max_window_size} characters: {transform} in ({string_repr})"
|
| 66 |
+
|
| 67 |
+
# restrict the vocabulary of predicates and transforms.
|
| 68 |
+
assert self.vocabulary.has_word(predicate), f"the predicate {predicate} has some symbols not in the vocabulary: {self.vocabulary}"
|
| 69 |
+
assert self.vocabulary.has_word(transform), f"the transform {transform} has some symbols not in the vocabulary: {self.vocabulary}"
|
| 70 |
+
|
| 71 |
+
self.predicate = predicate
|
| 72 |
+
self.transform = transform
|
| 73 |
+
|
| 74 |
+
return self
|
| 75 |
+
|
| 76 |
+
def __call__(self, inputs: list[str]) -> list[str]:
|
| 77 |
+
outputs: list[str] = []
|
| 78 |
+
for input in inputs:
|
| 79 |
+
output = input.replace(self.predicate, self.transform)
|
| 80 |
+
outputs.append(output)
|
| 81 |
+
|
| 82 |
+
return outputs
|
| 83 |
+
|
| 84 |
+
def __eq__(self, other):
|
| 85 |
+
return self.predicate == other.predicate and self.transform == other.transform
|
| 86 |
+
|
| 87 |
+
def __repr__(self):
|
| 88 |
+
return f'replace("{self.predicate}", "{self.transform}")'
|
| 89 |
+
|
| 90 |
+
def __str__(self):
|
| 91 |
+
return self.__repr__()
|
| 92 |
+
|
| 93 |
+
def identity(x: list[str]) -> list[str]: return x
|
| 94 |
+
|
| 95 |
+
def eval_outputs(
|
| 96 |
+
inputs: list[str], outputs: list[str],
|
| 97 |
+
pred_cascade: list[Union[str, ProgramBFCCNode]],
|
| 98 |
+
vocab_chars: str="abcdefghijkuvwxyz", max_programs: int=5):
|
| 99 |
+
pred_outputs: list[str] = [i for i in inputs]
|
| 100 |
+
valid_programs = 0
|
| 101 |
+
invalid_programs = 0
|
| 102 |
+
for program in pred_cascade[:max_programs]: # we should only pick the first max_programs (m in the paper) programs predicted by the model in the cascade (in case the model disobeys the instruction and uses more programs than max_programs)
|
| 103 |
+
if isinstance(program, str):
|
| 104 |
+
try:
|
| 105 |
+
program = ProgramBFCCNode.from_string(program, vocabulary=ProgramVocabulary(vocab_chars))
|
| 106 |
+
valid_programs += 1
|
| 107 |
+
except IndexError as e:
|
| 108 |
+
# print(f"\x1b[31;1m{e}\x1b[0m")
|
| 109 |
+
program = identity
|
| 110 |
+
invalid_programs += 1
|
| 111 |
+
except AssertionError as e:
|
| 112 |
+
# print(f"\x1b[31;1m{e}\x1b[0m")
|
| 113 |
+
program = identity # replace invalid programs with identity programs.
|
| 114 |
+
invalid_programs += 1
|
| 115 |
+
else:
|
| 116 |
+
# assert isinstance(program, ProgramBFCCNode), f"Invalid program type: {type(program)}"
|
| 117 |
+
# print(f"\x1b[31;1mInvalid program type: {type(program)}. Replacing with identity. Check if this isn't the result of WRONG FORMATTING!!!\x1b[0m")
|
| 118 |
+
program = identity
|
| 119 |
+
invalid_programs += 1
|
| 120 |
+
pred_outputs: list[str] = program(pred_outputs)
|
| 121 |
+
|
| 122 |
+
return pred_outputs, valid_programs, invalid_programs
|
| 123 |
+
|
| 124 |
+
def extract_last_python_block(markdown_text: str) -> str:
|
| 125 |
+
"""
|
| 126 |
+
Extracts first Python code block from a markdown string.
|
| 127 |
+
|
| 128 |
+
Args:
|
| 129 |
+
markdown_text (str): The markdown text containing code blocks.
|
| 130 |
+
|
| 131 |
+
Returns:
|
| 132 |
+
List[str]: A list of Python code blocks.
|
| 133 |
+
"""
|
| 134 |
+
code_block_pattern = r"```python\s(.*?)```"
|
| 135 |
+
matches = re.findall(code_block_pattern, markdown_text, re.DOTALL)
|
| 136 |
+
try: return [block.strip() for block in matches][-1]
|
| 137 |
+
except IndexError: return "[]"
|
| 138 |
+
|
| 139 |
+
def extract_program_str_from_last_python_block(model_response: str) -> str:
|
| 140 |
+
try: pred = extract_last_python_block(model_response)
|
| 141 |
+
except TypeError as e:
|
| 142 |
+
if model_response is None: pass
|
| 143 |
+
else: print(e)
|
| 144 |
+
pred = "replace('x','x')"
|
| 145 |
+
|
| 146 |
+
# wrap any replace(...) that isn’t already quoted in single quotes
|
| 147 |
+
# e.g. replace('kd','ka') → 'replace('kd','ka')'
|
| 148 |
+
pred = re.sub(
|
| 149 |
+
r"(?<!['\"])(replace\([^)]*\))(?!['\"])",
|
| 150 |
+
r'"\1"',
|
| 151 |
+
pred
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
return pred
|
| 155 |
+
|
| 156 |
+
def try_ast_literal_eval(string): # do some minor fixes.
|
| 157 |
+
"""Takes program sequence string and returns the extracted sequence of actual replace programs."""
|
| 158 |
+
try:
|
| 159 |
+
try:
|
| 160 |
+
program_seq = ast.literal_eval(string)
|
| 161 |
+
if isinstance(program_seq, str):
|
| 162 |
+
program_seq = [program_seq]
|
| 163 |
+
# print(program_seq, type(program_seq))
|
| 164 |
+
try:
|
| 165 |
+
assert isinstance(program_seq, list), f"program sequence of list form couldn't be extracted or derived: {program_seq}"
|
| 166 |
+
return program_seq
|
| 167 |
+
except AssertionError: return []
|
| 168 |
+
except ValueError: return []
|
| 169 |
+
except SyntaxError: return []
|
| 170 |
+
|
| 171 |
+
native=types.SimpleNamespace(eval_outputs=eval_outputs, try_ast_literal_eval=try_ast_literal_eval, extract_program_str_from_last_python_block=extract_program_str_from_last_python_block, ProgramVocabulary=ProgramVocabulary)
|
| 172 |
+
|
| 173 |
+
class PaperConstrainedNode(ProgramBFCCNode):
|
| 174 |
+
@classmethod
|
| 175 |
+
def from_string(cls,*args,**kwargs):
|
| 176 |
+
node=super().from_string(*args,**kwargs)
|
| 177 |
+
assert len(node.alpha)>0, 'paper forbids empty alpha'
|
| 178 |
+
return node
|
| 179 |
+
|
| 180 |
+
_globals=dict(native.eval_outputs.__globals__)
|
| 181 |
+
_globals['ProgramBFCCNode']=PaperConstrainedNode
|
| 182 |
+
eval_outputs=types.FunctionType(native.eval_outputs.__code__,_globals,'paper_eval_outputs',native.eval_outputs.__defaults__)
|
| 183 |
+
|
| 184 |
+
def extract(text):
|
| 185 |
+
return native.try_ast_literal_eval(native.extract_program_str_from_last_python_block(text))
|
| 186 |
+
|
| 187 |
+
def score(text,inputs,outputs):
|
| 188 |
+
cascade=extract(text)
|
| 189 |
+
predicted,valid,invalid=eval_outputs(inputs,outputs,cascade,VOCAB,5)
|
| 190 |
+
reasons=[]
|
| 191 |
+
for item in cascade[:5]:
|
| 192 |
+
if not isinstance(item,str): reasons.append('NON_STRING_RULE'); continue
|
| 193 |
+
try:
|
| 194 |
+
node=PaperConstrainedNode.from_string(item,vocabulary=native.ProgramVocabulary(VOCAB))
|
| 195 |
+
except (AssertionError,IndexError) as exc:
|
| 196 |
+
reasons.append(str(exc) or type(exc).__name__)
|
| 197 |
+
return {'pass_at_1':int(predicted==outputs),'predicted_outputs':predicted,'predicted_cascade':cascade,'valid_rules':valid,'invalid_rules':invalid,'invalid_rule_reasons':reasons,'complete_python_block_found':bool(re.search(r'```python\s(.*?)```',text,re.S)),'empty_cascade':not cascade}
|
data/analysis_contract.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"program_language": {
|
| 3 |
+
"alphabet": "abcdefghijkuvwxyz",
|
| 4 |
+
"lhs_lengths": [
|
| 5 |
+
1,
|
| 6 |
+
2,
|
| 7 |
+
3
|
| 8 |
+
],
|
| 9 |
+
"rhs_lengths": [
|
| 10 |
+
0,
|
| 11 |
+
1,
|
| 12 |
+
2,
|
| 13 |
+
3
|
| 14 |
+
],
|
| 15 |
+
"maximum_program_rules": 5,
|
| 16 |
+
"semantics": "Sequential Python str.replace, global non-overlapping replacement within each rule"
|
| 17 |
+
},
|
| 18 |
+
"model_labels": {
|
| 19 |
+
"M1": "Qwen3-8B",
|
| 20 |
+
"M2": "Olmo-3-7B-Think"
|
| 21 |
+
},
|
| 22 |
+
"generated_token_limit": 8192,
|
| 23 |
+
"family_count": 120,
|
| 24 |
+
"base_counts": [
|
| 25 |
+
3,
|
| 26 |
+
4,
|
| 27 |
+
5,
|
| 28 |
+
6
|
| 29 |
+
],
|
| 30 |
+
"new_trajectories": 1440,
|
| 31 |
+
"cached_base5_trajectories": 480,
|
| 32 |
+
"generation_seed_unit": "batch",
|
| 33 |
+
"selection": "Last complete eligible Python fence after the closed thinking delimiter; fall back to last complete list when no fence exists; fixed canonical rule notation mapping",
|
| 34 |
+
"statistics": {
|
| 35 |
+
"independent_unit": "family",
|
| 36 |
+
"bootstrap_replicates": 20000,
|
| 37 |
+
"rng": "NumPy PCG64",
|
| 38 |
+
"seed": 20261107,
|
| 39 |
+
"primary_cohort": "CERT_1_TO_2, separately at each base count",
|
| 40 |
+
"cross_base_cohort": "The 119 families certified at all four base counts",
|
| 41 |
+
"adjusted_comparisons": "Six model-by-new-base displayed-success differences in differences; Bonferroni percentile quantiles .05/12 and 1-.05/12",
|
| 42 |
+
"other_intervals": "Pointwise descriptive percentile intervals; quantiles .025 and .975"
|
| 43 |
+
},
|
| 44 |
+
"targets": {
|
| 45 |
+
"displayed": "The base-specific displayed specification",
|
| 46 |
+
"common9": "The original five base examples and both witness pairs; includes originally displayed examples",
|
| 47 |
+
"unshown_pair": "The two examples in the other witness pair, withheld from both conditions",
|
| 48 |
+
"original_unseen9": "Three originally unshown examples, excluding all original base examples and the displayed arm witness",
|
| 49 |
+
"broader": "32 UNIFORM and 32 ACTIVE inputs per family, fixed across all conditions and base counts"
|
| 50 |
+
},
|
| 51 |
+
"normalization": "Outcome text, generated token IDs, prompt token IDs, final selected programs, seeds and task identities are retained. Original machine paths, run timestamps and private operational metadata are removed. This package reconstructs scientific tables, not the original raw JSON byte layout or forward commits."
|
| 52 |
+
}
|
data/base_variants.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/broader_inputs.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/certificate_records.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/construction_audit.csv
ADDED
|
@@ -0,0 +1,481 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
sample_id,base_count,status,control_one_rule_count,separation_one_rule_count,base_nonidentity_count,new_base_activity,five_features_exact_match,candidate_checks,construction_seconds
|
| 2 |
+
5afeb1c4a228abe8751d0aff13ea1896ea3907e961e175fc43f83b5e0168e88c,3,SHORTCUT_RETURN_1_TO_1,2,1,1,,True,0,0
|
| 3 |
+
5afeb1c4a228abe8751d0aff13ea1896ea3907e961e175fc43f83b5e0168e88c,4,SHORTCUT_RETURN_1_TO_1,2,1,1,,True,0,0
|
| 4 |
+
5afeb1c4a228abe8751d0aff13ea1896ea3907e961e175fc43f83b5e0168e88c,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 5 |
+
5afeb1c4a228abe8751d0aff13ea1896ea3907e961e175fc43f83b5e0168e88c,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,1.4999881386756897e-05
|
| 6 |
+
e88652b6ae1dfe38f9c574ec4b018e63a7488c5f72fee1d009e69bd04b18705b,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 7 |
+
e88652b6ae1dfe38f9c574ec4b018e63a7488c5f72fee1d009e69bd04b18705b,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 8 |
+
e88652b6ae1dfe38f9c574ec4b018e63a7488c5f72fee1d009e69bd04b18705b,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 9 |
+
e88652b6ae1dfe38f9c574ec4b018e63a7488c5f72fee1d009e69bd04b18705b,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.200280040502548e-06
|
| 10 |
+
4f9a9215e8de5f90fdf5965943cf3862160629181ac1ddfbb7510ec624124437,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 11 |
+
4f9a9215e8de5f90fdf5965943cf3862160629181ac1ddfbb7510ec624124437,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 12 |
+
4f9a9215e8de5f90fdf5965943cf3862160629181ac1ddfbb7510ec624124437,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 13 |
+
4f9a9215e8de5f90fdf5965943cf3862160629181ac1ddfbb7510ec624124437,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,9.400304406881332e-06
|
| 14 |
+
8ac758bde59b461a3d5b9fe7b10e0f2aa691eacf4d98435c423223d98ed03859,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 15 |
+
8ac758bde59b461a3d5b9fe7b10e0f2aa691eacf4d98435c423223d98ed03859,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 16 |
+
8ac758bde59b461a3d5b9fe7b10e0f2aa691eacf4d98435c423223d98ed03859,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 17 |
+
8ac758bde59b461a3d5b9fe7b10e0f2aa691eacf4d98435c423223d98ed03859,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,6.899703294038773e-06
|
| 18 |
+
2498a96b060bd2314575fa6741d6ab0cf7519b1f9a6c0f34660cc1bdf792f4e6,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 19 |
+
2498a96b060bd2314575fa6741d6ab0cf7519b1f9a6c0f34660cc1bdf792f4e6,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 20 |
+
2498a96b060bd2314575fa6741d6ab0cf7519b1f9a6c0f34660cc1bdf792f4e6,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 21 |
+
2498a96b060bd2314575fa6741d6ab0cf7519b1f9a6c0f34660cc1bdf792f4e6,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,7.4999406933784485e-06
|
| 22 |
+
2f8e632f6a3d555a22d85739adffbc2d14a51a76379d6b2858bfbdfecf4407f5,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 23 |
+
2f8e632f6a3d555a22d85739adffbc2d14a51a76379d6b2858bfbdfecf4407f5,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 24 |
+
2f8e632f6a3d555a22d85739adffbc2d14a51a76379d6b2858bfbdfecf4407f5,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 25 |
+
2f8e632f6a3d555a22d85739adffbc2d14a51a76379d6b2858bfbdfecf4407f5,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.90015372633934e-06
|
| 26 |
+
6161817ec7ad4df81be536546bca3a8f356fe00f8144ddb43e780d9225a3e06f,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 27 |
+
6161817ec7ad4df81be536546bca3a8f356fe00f8144ddb43e780d9225a3e06f,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 28 |
+
6161817ec7ad4df81be536546bca3a8f356fe00f8144ddb43e780d9225a3e06f,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 29 |
+
6161817ec7ad4df81be536546bca3a8f356fe00f8144ddb43e780d9225a3e06f,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,7.4999406933784485e-06
|
| 30 |
+
51d04a36f9d72e5841a13a15d33f57d4501a8f0eab06ebd3ae212d95659dea79,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 31 |
+
51d04a36f9d72e5841a13a15d33f57d4501a8f0eab06ebd3ae212d95659dea79,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 32 |
+
51d04a36f9d72e5841a13a15d33f57d4501a8f0eab06ebd3ae212d95659dea79,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 33 |
+
51d04a36f9d72e5841a13a15d33f57d4501a8f0eab06ebd3ae212d95659dea79,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.6996941566467285e-06
|
| 34 |
+
f675f7367425f51a178273feed3394f2a26b722e259318d189b54b99760ecd5f,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 35 |
+
f675f7367425f51a178273feed3394f2a26b722e259318d189b54b99760ecd5f,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 36 |
+
f675f7367425f51a178273feed3394f2a26b722e259318d189b54b99760ecd5f,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 37 |
+
f675f7367425f51a178273feed3394f2a26b722e259318d189b54b99760ecd5f,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,7.700175046920776e-06
|
| 38 |
+
a6f4aea6333241339ec674d10a37bd15a1d06749452a42f47fc621feac756685,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 39 |
+
a6f4aea6333241339ec674d10a37bd15a1d06749452a42f47fc621feac756685,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 40 |
+
a6f4aea6333241339ec674d10a37bd15a1d06749452a42f47fc621feac756685,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 41 |
+
a6f4aea6333241339ec674d10a37bd15a1d06749452a42f47fc621feac756685,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.800051778554916e-06
|
| 42 |
+
80ecca2262225d596c8a4b4905ac09c37bc73f248acc374332cb8fdb250cae69,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 43 |
+
80ecca2262225d596c8a4b4905ac09c37bc73f248acc374332cb8fdb250cae69,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 44 |
+
80ecca2262225d596c8a4b4905ac09c37bc73f248acc374332cb8fdb250cae69,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 45 |
+
80ecca2262225d596c8a4b4905ac09c37bc73f248acc374332cb8fdb250cae69,6,CERT_1_TO_2,1,0,1,IDENTITY,True,3,8.999835699796677e-06
|
| 46 |
+
7235ea36738ce3e0a54c24649b5a3075902a920d4a2383647adbe588aee3a980,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 47 |
+
7235ea36738ce3e0a54c24649b5a3075902a920d4a2383647adbe588aee3a980,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 48 |
+
7235ea36738ce3e0a54c24649b5a3075902a920d4a2383647adbe588aee3a980,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 49 |
+
7235ea36738ce3e0a54c24649b5a3075902a920d4a2383647adbe588aee3a980,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.0996819734573364e-06
|
| 50 |
+
ce5852d092f6d10f0a1b5b5fb5baca2e314bd163effeb6bef3b7fee8aefd0ffe,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 51 |
+
ce5852d092f6d10f0a1b5b5fb5baca2e314bd163effeb6bef3b7fee8aefd0ffe,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 52 |
+
ce5852d092f6d10f0a1b5b5fb5baca2e314bd163effeb6bef3b7fee8aefd0ffe,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 53 |
+
ce5852d092f6d10f0a1b5b5fb5baca2e314bd163effeb6bef3b7fee8aefd0ffe,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.599802196025848e-06
|
| 54 |
+
3f40f5d0ce18cd567dc8f6ff707c925b0d27814907c650d3fd52af4c9c633d09,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 55 |
+
3f40f5d0ce18cd567dc8f6ff707c925b0d27814907c650d3fd52af4c9c633d09,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 56 |
+
3f40f5d0ce18cd567dc8f6ff707c925b0d27814907c650d3fd52af4c9c633d09,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 57 |
+
3f40f5d0ce18cd567dc8f6ff707c925b0d27814907c650d3fd52af4c9c633d09,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.100147634744644e-06
|
| 58 |
+
deb2458f8c124dbcfc98444c082b7ed7b0db129a479b60eee4c81ebd824ef090,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 59 |
+
deb2458f8c124dbcfc98444c082b7ed7b0db129a479b60eee4c81ebd824ef090,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 60 |
+
deb2458f8c124dbcfc98444c082b7ed7b0db129a479b60eee4c81ebd824ef090,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 61 |
+
deb2458f8c124dbcfc98444c082b7ed7b0db129a479b60eee4c81ebd824ef090,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.699919372797012e-06
|
| 62 |
+
78aef4f703d5b718ba52a07e065c227c9bd762159122c01b2f4674335f516d6f,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 63 |
+
78aef4f703d5b718ba52a07e065c227c9bd762159122c01b2f4674335f516d6f,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 64 |
+
78aef4f703d5b718ba52a07e065c227c9bd762159122c01b2f4674335f516d6f,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 65 |
+
78aef4f703d5b718ba52a07e065c227c9bd762159122c01b2f4674335f516d6f,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,3.899913281202316e-06
|
| 66 |
+
a77810b454ae1d284a979939c69179faa51ac69c2f98a713e15aa98f74e79dfd,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 67 |
+
a77810b454ae1d284a979939c69179faa51ac69c2f98a713e15aa98f74e79dfd,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 68 |
+
a77810b454ae1d284a979939c69179faa51ac69c2f98a713e15aa98f74e79dfd,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 69 |
+
a77810b454ae1d284a979939c69179faa51ac69c2f98a713e15aa98f74e79dfd,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.400033503770828e-06
|
| 70 |
+
4eb555f7dd71072f66e83b43dd5ab7fa666fbfc5ba00a0ed70a8f234ee731a80,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 71 |
+
4eb555f7dd71072f66e83b43dd5ab7fa666fbfc5ba00a0ed70a8f234ee731a80,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 72 |
+
4eb555f7dd71072f66e83b43dd5ab7fa666fbfc5ba00a0ed70a8f234ee731a80,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 73 |
+
4eb555f7dd71072f66e83b43dd5ab7fa666fbfc5ba00a0ed70a8f234ee731a80,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,4.500150680541992e-06
|
| 74 |
+
29747464799050476d2e4b5e2487e4195e79dcb5d9b3d148a77ddc9a1de592fa,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 75 |
+
29747464799050476d2e4b5e2487e4195e79dcb5d9b3d148a77ddc9a1de592fa,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 76 |
+
29747464799050476d2e4b5e2487e4195e79dcb5d9b3d148a77ddc9a1de592fa,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 77 |
+
29747464799050476d2e4b5e2487e4195e79dcb5d9b3d148a77ddc9a1de592fa,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,3.899913281202316e-06
|
| 78 |
+
d2dd0d881d683e7e2ad2c034cea769dce03e731bca78f6261cb3680ecad71eac,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 79 |
+
d2dd0d881d683e7e2ad2c034cea769dce03e731bca78f6261cb3680ecad71eac,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 80 |
+
d2dd0d881d683e7e2ad2c034cea769dce03e731bca78f6261cb3680ecad71eac,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 81 |
+
d2dd0d881d683e7e2ad2c034cea769dce03e731bca78f6261cb3680ecad71eac,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.8002769947052e-06
|
| 82 |
+
b9c62be1020460c4ae010c718ac3394000921301cc48c59498480b6d71294814,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 83 |
+
b9c62be1020460c4ae010c718ac3394000921301cc48c59498480b6d71294814,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 84 |
+
b9c62be1020460c4ae010c718ac3394000921301cc48c59498480b6d71294814,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 85 |
+
b9c62be1020460c4ae010c718ac3394000921301cc48c59498480b6d71294814,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.400033503770828e-06
|
| 86 |
+
6ef3b66634c9dce3fea2592255f2f512dda86a99abdc6570dd7c79eb09cdf6a6,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 87 |
+
6ef3b66634c9dce3fea2592255f2f512dda86a99abdc6570dd7c79eb09cdf6a6,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 88 |
+
6ef3b66634c9dce3fea2592255f2f512dda86a99abdc6570dd7c79eb09cdf6a6,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 89 |
+
6ef3b66634c9dce3fea2592255f2f512dda86a99abdc6570dd7c79eb09cdf6a6,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,4.600267857313156e-06
|
| 90 |
+
6c63f09c97e92b9a924bd615db2605093753dabc5e686c6f8e22483e6b6c9987,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 91 |
+
6c63f09c97e92b9a924bd615db2605093753dabc5e686c6f8e22483e6b6c9987,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 92 |
+
6c63f09c97e92b9a924bd615db2605093753dabc5e686c6f8e22483e6b6c9987,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 93 |
+
6c63f09c97e92b9a924bd615db2605093753dabc5e686c6f8e22483e6b6c9987,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.299916326999664e-06
|
| 94 |
+
cc6bb0322f2924df65a5a652ea7b2d797bd884cf66a8858e3707fcc8ed3665f6,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 95 |
+
cc6bb0322f2924df65a5a652ea7b2d797bd884cf66a8858e3707fcc8ed3665f6,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 96 |
+
cc6bb0322f2924df65a5a652ea7b2d797bd884cf66a8858e3707fcc8ed3665f6,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 97 |
+
cc6bb0322f2924df65a5a652ea7b2d797bd884cf66a8858e3707fcc8ed3665f6,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,8.400063961744308e-06
|
| 98 |
+
0c014001735373f48bb96f4d1021b715bc212cbd14823c8c7417f4953d513273,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 99 |
+
0c014001735373f48bb96f4d1021b715bc212cbd14823c8c7417f4953d513273,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 100 |
+
0c014001735373f48bb96f4d1021b715bc212cbd14823c8c7417f4953d513273,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 101 |
+
0c014001735373f48bb96f4d1021b715bc212cbd14823c8c7417f4953d513273,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.600042641162872e-06
|
| 102 |
+
fbdd34d00fef447111e36889e498168c6c1b0da89d1b35a45a800272c2739d7d,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 103 |
+
fbdd34d00fef447111e36889e498168c6c1b0da89d1b35a45a800272c2739d7d,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 104 |
+
fbdd34d00fef447111e36889e498168c6c1b0da89d1b35a45a800272c2739d7d,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 105 |
+
fbdd34d00fef447111e36889e498168c6c1b0da89d1b35a45a800272c2739d7d,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,7.200054824352264e-06
|
| 106 |
+
d2c08e1ebcc9ee1bcc1db838e588caa443be5d5666391a97e04045126b16668f,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 107 |
+
d2c08e1ebcc9ee1bcc1db838e588caa443be5d5666391a97e04045126b16668f,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 108 |
+
d2c08e1ebcc9ee1bcc1db838e588caa443be5d5666391a97e04045126b16668f,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 109 |
+
d2c08e1ebcc9ee1bcc1db838e588caa443be5d5666391a97e04045126b16668f,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.1998143792152405e-06
|
| 110 |
+
ca33a6c7fe176351a90f9990b595395d4a8de2a36b9234d9f716f9b99ae52ee1,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 111 |
+
ca33a6c7fe176351a90f9990b595395d4a8de2a36b9234d9f716f9b99ae52ee1,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 112 |
+
ca33a6c7fe176351a90f9990b595395d4a8de2a36b9234d9f716f9b99ae52ee1,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 113 |
+
ca33a6c7fe176351a90f9990b595395d4a8de2a36b9234d9f716f9b99ae52ee1,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,8.699949830770493e-06
|
| 114 |
+
4271134b0e14f43851a2c5a9629f9228d411b2b5bea90948dfa41373968b6d6e,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 115 |
+
4271134b0e14f43851a2c5a9629f9228d411b2b5bea90948dfa41373968b6d6e,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 116 |
+
4271134b0e14f43851a2c5a9629f9228d411b2b5bea90948dfa41373968b6d6e,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 117 |
+
4271134b0e14f43851a2c5a9629f9228d411b2b5bea90948dfa41373968b6d6e,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.299931555986404e-06
|
| 118 |
+
849332c9214ba8149f78cc7b610f152708a2d08ea5fbd06052238d3a68368f26,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 119 |
+
849332c9214ba8149f78cc7b610f152708a2d08ea5fbd06052238d3a68368f26,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 120 |
+
849332c9214ba8149f78cc7b610f152708a2d08ea5fbd06052238d3a68368f26,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 121 |
+
849332c9214ba8149f78cc7b610f152708a2d08ea5fbd06052238d3a68368f26,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.299916326999664e-06
|
| 122 |
+
31bc531191645d0bd3b82dfae8c574e04abb59542b15c97062a4eeed292ef48e,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 123 |
+
31bc531191645d0bd3b82dfae8c574e04abb59542b15c97062a4eeed292ef48e,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 124 |
+
31bc531191645d0bd3b82dfae8c574e04abb59542b15c97062a4eeed292ef48e,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 125 |
+
31bc531191645d0bd3b82dfae8c574e04abb59542b15c97062a4eeed292ef48e,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.400033503770828e-06
|
| 126 |
+
4181f8341f03fcb9e73d74cd47a0020f3f4119da18f836427815420ddbf24a33,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 127 |
+
4181f8341f03fcb9e73d74cd47a0020f3f4119da18f836427815420ddbf24a33,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 128 |
+
4181f8341f03fcb9e73d74cd47a0020f3f4119da18f836427815420ddbf24a33,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 129 |
+
4181f8341f03fcb9e73d74cd47a0020f3f4119da18f836427815420ddbf24a33,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.699919372797012e-06
|
| 130 |
+
1a0c3fca1b2671f50b0ee9caf2c4a2bc8cf14d2f2df86b95f0ec7b842be9f4dc,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 131 |
+
1a0c3fca1b2671f50b0ee9caf2c4a2bc8cf14d2f2df86b95f0ec7b842be9f4dc,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 132 |
+
1a0c3fca1b2671f50b0ee9caf2c4a2bc8cf14d2f2df86b95f0ec7b842be9f4dc,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 133 |
+
1a0c3fca1b2671f50b0ee9caf2c4a2bc8cf14d2f2df86b95f0ec7b842be9f4dc,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,3.600027412176132e-06
|
| 134 |
+
6bd41d8c5cf0e226fea307fc19ea6e00b6c2335accf0df60f5e57733a1cd2ac6,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 135 |
+
6bd41d8c5cf0e226fea307fc19ea6e00b6c2335accf0df60f5e57733a1cd2ac6,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 136 |
+
6bd41d8c5cf0e226fea307fc19ea6e00b6c2335accf0df60f5e57733a1cd2ac6,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 137 |
+
6bd41d8c5cf0e226fea307fc19ea6e00b6c2335accf0df60f5e57733a1cd2ac6,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.1997991502285e-06
|
| 138 |
+
2853b5634bb43d58cb6ab7e177ab13e5e4208fbc0d9706767e24d9731df11fad,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 139 |
+
2853b5634bb43d58cb6ab7e177ab13e5e4208fbc0d9706767e24d9731df11fad,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 140 |
+
2853b5634bb43d58cb6ab7e177ab13e5e4208fbc0d9706767e24d9731df11fad,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 141 |
+
2853b5634bb43d58cb6ab7e177ab13e5e4208fbc0d9706767e24d9731df11fad,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,4.70038503408432e-06
|
| 142 |
+
4abba5cd3270f86c4ee5b5db5269fea4e79f1216290f205457ad797f6532f779,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 143 |
+
4abba5cd3270f86c4ee5b5db5269fea4e79f1216290f205457ad797f6532f779,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 144 |
+
4abba5cd3270f86c4ee5b5db5269fea4e79f1216290f205457ad797f6532f779,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 145 |
+
4abba5cd3270f86c4ee5b5db5269fea4e79f1216290f205457ad797f6532f779,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,0.0002506999298930168
|
| 146 |
+
3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 147 |
+
3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 148 |
+
3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 149 |
+
3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.800036549568176e-06
|
| 150 |
+
173f66d7fb2694c623c2abd8e8c20df48e82136d6c27ffe91e94612798ac69eb,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 151 |
+
173f66d7fb2694c623c2abd8e8c20df48e82136d6c27ffe91e94612798ac69eb,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 152 |
+
173f66d7fb2694c623c2abd8e8c20df48e82136d6c27ffe91e94612798ac69eb,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 153 |
+
173f66d7fb2694c623c2abd8e8c20df48e82136d6c27ffe91e94612798ac69eb,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.200264811515808e-06
|
| 154 |
+
abafad43bb4177b3fa9c15fb8898492903f7fae570b4b157e8d7e47c73fb9269,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 155 |
+
abafad43bb4177b3fa9c15fb8898492903f7fae570b4b157e8d7e47c73fb9269,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 156 |
+
abafad43bb4177b3fa9c15fb8898492903f7fae570b4b157e8d7e47c73fb9269,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 157 |
+
abafad43bb4177b3fa9c15fb8898492903f7fae570b4b157e8d7e47c73fb9269,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.200280040502548e-06
|
| 158 |
+
5678980e464d4d9bc972812c75e1c17e366e70a328de2eb362e4a45f7263d3cc,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 159 |
+
5678980e464d4d9bc972812c75e1c17e366e70a328de2eb362e4a45f7263d3cc,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 160 |
+
5678980e464d4d9bc972812c75e1c17e366e70a328de2eb362e4a45f7263d3cc,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 161 |
+
5678980e464d4d9bc972812c75e1c17e366e70a328de2eb362e4a45f7263d3cc,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,5.700159817934036e-06
|
| 162 |
+
1779fc40895393da5eb25f55c29543576291e6e22e3a2385091f01a020abb45e,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 163 |
+
1779fc40895393da5eb25f55c29543576291e6e22e3a2385091f01a020abb45e,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 164 |
+
1779fc40895393da5eb25f55c29543576291e6e22e3a2385091f01a020abb45e,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 165 |
+
1779fc40895393da5eb25f55c29543576291e6e22e3a2385091f01a020abb45e,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.899928510189056e-06
|
| 166 |
+
d01bcbd0aa743ca02010ed98061975389c4cb3405bc2105dbb935f5951388184,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 167 |
+
d01bcbd0aa743ca02010ed98061975389c4cb3405bc2105dbb935f5951388184,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 168 |
+
d01bcbd0aa743ca02010ed98061975389c4cb3405bc2105dbb935f5951388184,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 169 |
+
d01bcbd0aa743ca02010ed98061975389c4cb3405bc2105dbb935f5951388184,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.1997991502285e-06
|
| 170 |
+
c3dd49c12f83410dc0ac323df63f316086bb63a836ad9809d9efb5ec6cf9e0d0,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 171 |
+
c3dd49c12f83410dc0ac323df63f316086bb63a836ad9809d9efb5ec6cf9e0d0,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 172 |
+
c3dd49c12f83410dc0ac323df63f316086bb63a836ad9809d9efb5ec6cf9e0d0,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 173 |
+
c3dd49c12f83410dc0ac323df63f316086bb63a836ad9809d9efb5ec6cf9e0d0,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.6996941566467285e-06
|
| 174 |
+
60ee40ba26e4817ece08ddfbac06d68dfbfee9a9139739016b4ff642933d9b73,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 175 |
+
60ee40ba26e4817ece08ddfbac06d68dfbfee9a9139739016b4ff642933d9b73,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 176 |
+
60ee40ba26e4817ece08ddfbac06d68dfbfee9a9139739016b4ff642933d9b73,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 177 |
+
60ee40ba26e4817ece08ddfbac06d68dfbfee9a9139739016b4ff642933d9b73,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,5.500391125679016e-06
|
| 178 |
+
af45c30e942d7934baa54af49e8ed0db11f2506b55bdd4f1429baad0b700453b,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 179 |
+
af45c30e942d7934baa54af49e8ed0db11f2506b55bdd4f1429baad0b700453b,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 180 |
+
af45c30e942d7934baa54af49e8ed0db11f2506b55bdd4f1429baad0b700453b,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 181 |
+
af45c30e942d7934baa54af49e8ed0db11f2506b55bdd4f1429baad0b700453b,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.400033503770828e-06
|
| 182 |
+
f15a6ae8e7e8d4b1b5f085d150436773560c24eac0ececc1ca84c26aeea30718,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 183 |
+
f15a6ae8e7e8d4b1b5f085d150436773560c24eac0ececc1ca84c26aeea30718,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 184 |
+
f15a6ae8e7e8d4b1b5f085d150436773560c24eac0ececc1ca84c26aeea30718,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 185 |
+
f15a6ae8e7e8d4b1b5f085d150436773560c24eac0ececc1ca84c26aeea30718,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,3.899913281202316e-06
|
| 186 |
+
cd7bf1f19e95d2e44c6c28099fd9d51b1dbf991d40c58865382a571099d3dfc6,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 187 |
+
cd7bf1f19e95d2e44c6c28099fd9d51b1dbf991d40c58865382a571099d3dfc6,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 188 |
+
cd7bf1f19e95d2e44c6c28099fd9d51b1dbf991d40c58865382a571099d3dfc6,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 189 |
+
cd7bf1f19e95d2e44c6c28099fd9d51b1dbf991d40c58865382a571099d3dfc6,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.400033503770828e-06
|
| 190 |
+
8fd2ab966a06423e53b930c0d597e68ee3808b16c35378cfd70581494ca67f5c,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 191 |
+
8fd2ab966a06423e53b930c0d597e68ee3808b16c35378cfd70581494ca67f5c,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 192 |
+
8fd2ab966a06423e53b930c0d597e68ee3808b16c35378cfd70581494ca67f5c,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 193 |
+
8fd2ab966a06423e53b930c0d597e68ee3808b16c35378cfd70581494ca67f5c,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.299931555986404e-06
|
| 194 |
+
6218ce9545da667042639e864134570b04aacea76b8ccd5f7a8813d6738f7a8f,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 195 |
+
6218ce9545da667042639e864134570b04aacea76b8ccd5f7a8813d6738f7a8f,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 196 |
+
6218ce9545da667042639e864134570b04aacea76b8ccd5f7a8813d6738f7a8f,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 197 |
+
6218ce9545da667042639e864134570b04aacea76b8ccd5f7a8813d6738f7a8f,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,6.400048732757568e-06
|
| 198 |
+
83028807ffe1aae7a8360b7a0068d9a67e80e9e0a9c3d932c9e7b61438a2a5dd,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 199 |
+
83028807ffe1aae7a8360b7a0068d9a67e80e9e0a9c3d932c9e7b61438a2a5dd,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 200 |
+
83028807ffe1aae7a8360b7a0068d9a67e80e9e0a9c3d932c9e7b61438a2a5dd,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 201 |
+
83028807ffe1aae7a8360b7a0068d9a67e80e9e0a9c3d932c9e7b61438a2a5dd,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,6.0996972024440765e-06
|
| 202 |
+
bb2092f0b0f5b3c98b8030a9243f5daacff895e8f5c3bc35e45021c2023a513b,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 203 |
+
bb2092f0b0f5b3c98b8030a9243f5daacff895e8f5c3bc35e45021c2023a513b,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 204 |
+
bb2092f0b0f5b3c98b8030a9243f5daacff895e8f5c3bc35e45021c2023a513b,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 205 |
+
bb2092f0b0f5b3c98b8030a9243f5daacff895e8f5c3bc35e45021c2023a513b,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,3.600027412176132e-06
|
| 206 |
+
93f4e84964ef7be7f62d786004db86713af7f0053bca5083dcdc81969cc70a71,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 207 |
+
93f4e84964ef7be7f62d786004db86713af7f0053bca5083dcdc81969cc70a71,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 208 |
+
93f4e84964ef7be7f62d786004db86713af7f0053bca5083dcdc81969cc70a71,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 209 |
+
93f4e84964ef7be7f62d786004db86713af7f0053bca5083dcdc81969cc70a71,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,4.500150680541992e-06
|
| 210 |
+
31c0b8a636daa2f1910cd6e58495234da2304e5037d738f2f9e0b45e045c3746,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 211 |
+
31c0b8a636daa2f1910cd6e58495234da2304e5037d738f2f9e0b45e045c3746,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 212 |
+
31c0b8a636daa2f1910cd6e58495234da2304e5037d738f2f9e0b45e045c3746,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 213 |
+
31c0b8a636daa2f1910cd6e58495234da2304e5037d738f2f9e0b45e045c3746,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,3.700144588947296e-06
|
| 214 |
+
fe092440b96ec3f34cf9187558dc058e5e542f44e200f484877c00b11368b993,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 215 |
+
fe092440b96ec3f34cf9187558dc058e5e542f44e200f484877c00b11368b993,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 216 |
+
fe092440b96ec3f34cf9187558dc058e5e542f44e200f484877c00b11368b993,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 217 |
+
fe092440b96ec3f34cf9187558dc058e5e542f44e200f484877c00b11368b993,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.500150680541992e-06
|
| 218 |
+
5e3613673d3c55a3c88f8e7283cd683a77b867ac033a7a3bea7dad7a75e45058,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 219 |
+
5e3613673d3c55a3c88f8e7283cd683a77b867ac033a7a3bea7dad7a75e45058,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 220 |
+
5e3613673d3c55a3c88f8e7283cd683a77b867ac033a7a3bea7dad7a75e45058,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 221 |
+
5e3613673d3c55a3c88f8e7283cd683a77b867ac033a7a3bea7dad7a75e45058,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,3.600027412176132e-06
|
| 222 |
+
22e75adb49a83eff155cac9f86f62d4805a2de88165bad263a49c16c9fd46051,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 223 |
+
22e75adb49a83eff155cac9f86f62d4805a2de88165bad263a49c16c9fd46051,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 224 |
+
22e75adb49a83eff155cac9f86f62d4805a2de88165bad263a49c16c9fd46051,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 225 |
+
22e75adb49a83eff155cac9f86f62d4805a2de88165bad263a49c16c9fd46051,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.899703294038773e-06
|
| 226 |
+
ef085077c063e083b3597dd0e6147ed0877de735ba0dd9628da1393c731baa7a,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 227 |
+
ef085077c063e083b3597dd0e6147ed0877de735ba0dd9628da1393c731baa7a,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 228 |
+
ef085077c063e083b3597dd0e6147ed0877de735ba0dd9628da1393c731baa7a,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 229 |
+
ef085077c063e083b3597dd0e6147ed0877de735ba0dd9628da1393c731baa7a,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,8.299946784973145e-06
|
| 230 |
+
11239b8c636f82b4c140f14a89b409338a2d19b991fb0922d623c12f7d085b05,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 231 |
+
11239b8c636f82b4c140f14a89b409338a2d19b991fb0922d623c12f7d085b05,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 232 |
+
11239b8c636f82b4c140f14a89b409338a2d19b991fb0922d623c12f7d085b05,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 233 |
+
11239b8c636f82b4c140f14a89b409338a2d19b991fb0922d623c12f7d085b05,6,CERT_1_TO_2,2,0,1,IDENTITY,True,2,6.400048732757568e-06
|
| 234 |
+
5cf065d3cdb2ebe090acb081963bdc08c5a613d9188174cf495eebb9e9e3fae1,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 235 |
+
5cf065d3cdb2ebe090acb081963bdc08c5a613d9188174cf495eebb9e9e3fae1,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 236 |
+
5cf065d3cdb2ebe090acb081963bdc08c5a613d9188174cf495eebb9e9e3fae1,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 237 |
+
5cf065d3cdb2ebe090acb081963bdc08c5a613d9188174cf495eebb9e9e3fae1,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.900394171476364e-06
|
| 238 |
+
aca1382576b360521bc91a4e1f698243701f654a1ad69b25059b5b549161d974,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 239 |
+
aca1382576b360521bc91a4e1f698243701f654a1ad69b25059b5b549161d974,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 240 |
+
aca1382576b360521bc91a4e1f698243701f654a1ad69b25059b5b549161d974,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 241 |
+
aca1382576b360521bc91a4e1f698243701f654a1ad69b25059b5b549161d974,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,3.899913281202316e-06
|
| 242 |
+
300b5ddca63038b12a84456e3369d578ff98bb35687d3042d5b425ee4270363c,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 243 |
+
300b5ddca63038b12a84456e3369d578ff98bb35687d3042d5b425ee4270363c,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 244 |
+
300b5ddca63038b12a84456e3369d578ff98bb35687d3042d5b425ee4270363c,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 245 |
+
300b5ddca63038b12a84456e3369d578ff98bb35687d3042d5b425ee4270363c,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.1998143792152405e-06
|
| 246 |
+
4e45d55551dbade75c372168406be3f6831d0961334fee18564837eb0ed57b55,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 247 |
+
4e45d55551dbade75c372168406be3f6831d0961334fee18564837eb0ed57b55,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 248 |
+
4e45d55551dbade75c372168406be3f6831d0961334fee18564837eb0ed57b55,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 249 |
+
4e45d55551dbade75c372168406be3f6831d0961334fee18564837eb0ed57b55,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.899928510189056e-06
|
| 250 |
+
f8f589b68ee6d46ed00c5b98d1cdd368be6a2fb34a520311d2956840c773017b,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 251 |
+
f8f589b68ee6d46ed00c5b98d1cdd368be6a2fb34a520311d2956840c773017b,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 252 |
+
f8f589b68ee6d46ed00c5b98d1cdd368be6a2fb34a520311d2956840c773017b,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 253 |
+
f8f589b68ee6d46ed00c5b98d1cdd368be6a2fb34a520311d2956840c773017b,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,4.800036549568176e-06
|
| 254 |
+
cbe36a26b056a428ba13954013c62860088baaf774f7f82abf684984d9ff41b2,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 255 |
+
cbe36a26b056a428ba13954013c62860088baaf774f7f82abf684984d9ff41b2,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 256 |
+
cbe36a26b056a428ba13954013c62860088baaf774f7f82abf684984d9ff41b2,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 257 |
+
cbe36a26b056a428ba13954013c62860088baaf774f7f82abf684984d9ff41b2,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.500150680541992e-06
|
| 258 |
+
e0a0847e949e573486d195c93532f96165ebce309d22c05b05cdd278718b53b6,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 259 |
+
e0a0847e949e573486d195c93532f96165ebce309d22c05b05cdd278718b53b6,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 260 |
+
e0a0847e949e573486d195c93532f96165ebce309d22c05b05cdd278718b53b6,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 261 |
+
e0a0847e949e573486d195c93532f96165ebce309d22c05b05cdd278718b53b6,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.1997991502285e-06
|
| 262 |
+
3d200bdd821dd0a9d16ec6f06049d0b6f9eac6a91d45dd328b5d1a150dd8a4a3,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 263 |
+
3d200bdd821dd0a9d16ec6f06049d0b6f9eac6a91d45dd328b5d1a150dd8a4a3,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 264 |
+
3d200bdd821dd0a9d16ec6f06049d0b6f9eac6a91d45dd328b5d1a150dd8a4a3,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 265 |
+
3d200bdd821dd0a9d16ec6f06049d0b6f9eac6a91d45dd328b5d1a150dd8a4a3,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,7.999595254659653e-06
|
| 266 |
+
a512ae7af7577b8849c8f1438a4d89330ea4b506bae452735a4307d13f79b184,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 267 |
+
a512ae7af7577b8849c8f1438a4d89330ea4b506bae452735a4307d13f79b184,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 268 |
+
a512ae7af7577b8849c8f1438a4d89330ea4b506bae452735a4307d13f79b184,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 269 |
+
a512ae7af7577b8849c8f1438a4d89330ea4b506bae452735a4307d13f79b184,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,3.899913281202316e-06
|
| 270 |
+
6676811bd8013e31c30f66329b7a1d9786439a79efea4370c2b47de7e55f1e0e,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 271 |
+
6676811bd8013e31c30f66329b7a1d9786439a79efea4370c2b47de7e55f1e0e,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 272 |
+
6676811bd8013e31c30f66329b7a1d9786439a79efea4370c2b47de7e55f1e0e,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 273 |
+
6676811bd8013e31c30f66329b7a1d9786439a79efea4370c2b47de7e55f1e0e,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.299916326999664e-06
|
| 274 |
+
907c0892581c3ee8a7b39c134b83280d1c40acebb2f88af12477e38e1cb57470,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 275 |
+
907c0892581c3ee8a7b39c134b83280d1c40acebb2f88af12477e38e1cb57470,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 276 |
+
907c0892581c3ee8a7b39c134b83280d1c40acebb2f88af12477e38e1cb57470,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 277 |
+
907c0892581c3ee8a7b39c134b83280d1c40acebb2f88af12477e38e1cb57470,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,7.999595254659653e-06
|
| 278 |
+
b5e984a820dd97486eabfe810b5f935af91544ba9596ecf5ce0fc08fbed2301b,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 279 |
+
b5e984a820dd97486eabfe810b5f935af91544ba9596ecf5ce0fc08fbed2301b,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 280 |
+
b5e984a820dd97486eabfe810b5f935af91544ba9596ecf5ce0fc08fbed2301b,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 281 |
+
b5e984a820dd97486eabfe810b5f935af91544ba9596ecf5ce0fc08fbed2301b,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,6.00004568696022e-06
|
| 282 |
+
4a6a24c95a70b366b1ae94fdf8f92934ba9249c95d3a490bb6a5923798c85819,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 283 |
+
4a6a24c95a70b366b1ae94fdf8f92934ba9249c95d3a490bb6a5923798c85819,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 284 |
+
4a6a24c95a70b366b1ae94fdf8f92934ba9249c95d3a490bb6a5923798c85819,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 285 |
+
4a6a24c95a70b366b1ae94fdf8f92934ba9249c95d3a490bb6a5923798c85819,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.7998113334178925e-06
|
| 286 |
+
9b97bc055a30d7669e96a0e260d18cc71cd1c0604ea1f15861ae0d681851a6fb,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 287 |
+
9b97bc055a30d7669e96a0e260d18cc71cd1c0604ea1f15861ae0d681851a6fb,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 288 |
+
9b97bc055a30d7669e96a0e260d18cc71cd1c0604ea1f15861ae0d681851a6fb,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 289 |
+
9b97bc055a30d7669e96a0e260d18cc71cd1c0604ea1f15861ae0d681851a6fb,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,1.3699755072593689e-05
|
| 290 |
+
4cf6758635b2966e1870c0656c884474a2fa3833e3fccaac0c6d5a6562047445,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 291 |
+
4cf6758635b2966e1870c0656c884474a2fa3833e3fccaac0c6d5a6562047445,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 292 |
+
4cf6758635b2966e1870c0656c884474a2fa3833e3fccaac0c6d5a6562047445,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 293 |
+
4cf6758635b2966e1870c0656c884474a2fa3833e3fccaac0c6d5a6562047445,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.6999346017837524e-06
|
| 294 |
+
c4063d4ae0a32febb91149468bc80236f4a4f8ae2654d9dedf61c51d206d939a,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 295 |
+
c4063d4ae0a32febb91149468bc80236f4a4f8ae2654d9dedf61c51d206d939a,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 296 |
+
c4063d4ae0a32febb91149468bc80236f4a4f8ae2654d9dedf61c51d206d939a,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 297 |
+
c4063d4ae0a32febb91149468bc80236f4a4f8ae2654d9dedf61c51d206d939a,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.899928510189056e-06
|
| 298 |
+
6973a3d5bfcd74d0becedcd1833f0509bd10cddec62f2eaece093f085dd89747,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 299 |
+
6973a3d5bfcd74d0becedcd1833f0509bd10cddec62f2eaece093f085dd89747,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 300 |
+
6973a3d5bfcd74d0becedcd1833f0509bd10cddec62f2eaece093f085dd89747,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 301 |
+
6973a3d5bfcd74d0becedcd1833f0509bd10cddec62f2eaece093f085dd89747,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.400048732757568e-06
|
| 302 |
+
af0951734917891d1d7c53f90f7a9da76452ec128f0f74cef523dd369baf89ea,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 303 |
+
af0951734917891d1d7c53f90f7a9da76452ec128f0f74cef523dd369baf89ea,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 304 |
+
af0951734917891d1d7c53f90f7a9da76452ec128f0f74cef523dd369baf89ea,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 305 |
+
af0951734917891d1d7c53f90f7a9da76452ec128f0f74cef523dd369baf89ea,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.0996972024440765e-06
|
| 306 |
+
c424e51989d30e9c383f54519229a709f98f1d12352d97329f4c4dcfafeb647a,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 307 |
+
c424e51989d30e9c383f54519229a709f98f1d12352d97329f4c4dcfafeb647a,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 308 |
+
c424e51989d30e9c383f54519229a709f98f1d12352d97329f4c4dcfafeb647a,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 309 |
+
c424e51989d30e9c383f54519229a709f98f1d12352d97329f4c4dcfafeb647a,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.200039595365524e-06
|
| 310 |
+
e1434beaae427089f40f0429f1c8eb3acfff9d8bc7b6f4fb6d85246ea7fbc7a2,3,CERT_1_TO_2,3,0,1,,True,0,0
|
| 311 |
+
e1434beaae427089f40f0429f1c8eb3acfff9d8bc7b6f4fb6d85246ea7fbc7a2,4,CERT_1_TO_2,3,0,1,,True,0,0
|
| 312 |
+
e1434beaae427089f40f0429f1c8eb3acfff9d8bc7b6f4fb6d85246ea7fbc7a2,5,CERT_1_TO_2,3,0,1,,True,0,0
|
| 313 |
+
e1434beaae427089f40f0429f1c8eb3acfff9d8bc7b6f4fb6d85246ea7fbc7a2,6,CERT_1_TO_2,3,0,1,IDENTITY,True,1,4.500150680541992e-06
|
| 314 |
+
9ca6df982614ba893bd88246d413222695cdc459e08a14a62d4db3543a83e700,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 315 |
+
9ca6df982614ba893bd88246d413222695cdc459e08a14a62d4db3543a83e700,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 316 |
+
9ca6df982614ba893bd88246d413222695cdc459e08a14a62d4db3543a83e700,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 317 |
+
9ca6df982614ba893bd88246d413222695cdc459e08a14a62d4db3543a83e700,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,4.299916326999664e-06
|
| 318 |
+
9ecb04df6eecca22d3e34ca14355fa783507e1473696cbef6f009ec5ad02be56,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 319 |
+
9ecb04df6eecca22d3e34ca14355fa783507e1473696cbef6f009ec5ad02be56,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 320 |
+
9ecb04df6eecca22d3e34ca14355fa783507e1473696cbef6f009ec5ad02be56,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 321 |
+
9ecb04df6eecca22d3e34ca14355fa783507e1473696cbef6f009ec5ad02be56,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.100147634744644e-06
|
| 322 |
+
fa65d93a95e80cbea45a3b9f87b8f33415157d63ca1437556cb9d983a8cafc63,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 323 |
+
fa65d93a95e80cbea45a3b9f87b8f33415157d63ca1437556cb9d983a8cafc63,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 324 |
+
fa65d93a95e80cbea45a3b9f87b8f33415157d63ca1437556cb9d983a8cafc63,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 325 |
+
fa65d93a95e80cbea45a3b9f87b8f33415157d63ca1437556cb9d983a8cafc63,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.70038503408432e-06
|
| 326 |
+
f0813d4d64f3d84772014f98368edb9a363727159a44b58974cf78338345a4ed,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 327 |
+
f0813d4d64f3d84772014f98368edb9a363727159a44b58974cf78338345a4ed,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 328 |
+
f0813d4d64f3d84772014f98368edb9a363727159a44b58974cf78338345a4ed,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 329 |
+
f0813d4d64f3d84772014f98368edb9a363727159a44b58974cf78338345a4ed,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.899928510189056e-06
|
| 330 |
+
f1043ce7f4a9ff83a54020a860132d1a923a64d9149520b39198a4cad1def347,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 331 |
+
f1043ce7f4a9ff83a54020a860132d1a923a64d9149520b39198a4cad1def347,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 332 |
+
f1043ce7f4a9ff83a54020a860132d1a923a64d9149520b39198a4cad1def347,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 333 |
+
f1043ce7f4a9ff83a54020a860132d1a923a64d9149520b39198a4cad1def347,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.90015372633934e-06
|
| 334 |
+
1d41225f7479cc6bbc0587ce6ddbfeffce4b039160ad46b3b9fa1bca26479502,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 335 |
+
1d41225f7479cc6bbc0587ce6ddbfeffce4b039160ad46b3b9fa1bca26479502,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 336 |
+
1d41225f7479cc6bbc0587ce6ddbfeffce4b039160ad46b3b9fa1bca26479502,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 337 |
+
1d41225f7479cc6bbc0587ce6ddbfeffce4b039160ad46b3b9fa1bca26479502,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.999805241823196e-06
|
| 338 |
+
9a136f33b317a65d5efe952fb7d5255042a3055a2a1530d19c5fb036f525f848,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 339 |
+
9a136f33b317a65d5efe952fb7d5255042a3055a2a1530d19c5fb036f525f848,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 340 |
+
9a136f33b317a65d5efe952fb7d5255042a3055a2a1530d19c5fb036f525f848,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 341 |
+
9a136f33b317a65d5efe952fb7d5255042a3055a2a1530d19c5fb036f525f848,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,7.899943739175797e-06
|
| 342 |
+
413883cd4097ed5b87c5ef7983637c9b972c2484179e692739531de80d852d4c,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 343 |
+
413883cd4097ed5b87c5ef7983637c9b972c2484179e692739531de80d852d4c,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 344 |
+
413883cd4097ed5b87c5ef7983637c9b972c2484179e692739531de80d852d4c,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 345 |
+
413883cd4097ed5b87c5ef7983637c9b972c2484179e692739531de80d852d4c,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.300397217273712e-06
|
| 346 |
+
b35fb93822b43380c5f2f57cf99c17e70d5d4978bf3546d21f2989b2a7d61374,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 347 |
+
b35fb93822b43380c5f2f57cf99c17e70d5d4978bf3546d21f2989b2a7d61374,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 348 |
+
b35fb93822b43380c5f2f57cf99c17e70d5d4978bf3546d21f2989b2a7d61374,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 349 |
+
b35fb93822b43380c5f2f57cf99c17e70d5d4978bf3546d21f2989b2a7d61374,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,8.19982960820198e-06
|
| 350 |
+
d14de179f71df7a1f52f82fdbcb8503b0d21a521099d95444f7bf52d1cb272c9,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 351 |
+
d14de179f71df7a1f52f82fdbcb8503b0d21a521099d95444f7bf52d1cb272c9,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 352 |
+
d14de179f71df7a1f52f82fdbcb8503b0d21a521099d95444f7bf52d1cb272c9,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 353 |
+
d14de179f71df7a1f52f82fdbcb8503b0d21a521099d95444f7bf52d1cb272c9,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,8.500181138515472e-06
|
| 354 |
+
b8759c3251a13e89d55ab4f94d9a6c1880f4813ba7b90fde8a3e2ab2d6d4f27c,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 355 |
+
b8759c3251a13e89d55ab4f94d9a6c1880f4813ba7b90fde8a3e2ab2d6d4f27c,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 356 |
+
b8759c3251a13e89d55ab4f94d9a6c1880f4813ba7b90fde8a3e2ab2d6d4f27c,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 357 |
+
b8759c3251a13e89d55ab4f94d9a6c1880f4813ba7b90fde8a3e2ab2d6d4f27c,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.100162863731384e-06
|
| 358 |
+
c15f41eac9c1efc526d75a8f8adafc01b029d9cdb18bc198ad604bbc6471e4c9,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 359 |
+
c15f41eac9c1efc526d75a8f8adafc01b029d9cdb18bc198ad604bbc6471e4c9,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 360 |
+
c15f41eac9c1efc526d75a8f8adafc01b029d9cdb18bc198ad604bbc6471e4c9,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 361 |
+
c15f41eac9c1efc526d75a8f8adafc01b029d9cdb18bc198ad604bbc6471e4c9,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,6.100162863731384e-06
|
| 362 |
+
54fee0f8bc984848c7d80af120befb83dc9bc79dcff4f6be15b7330f66a9a2c8,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 363 |
+
54fee0f8bc984848c7d80af120befb83dc9bc79dcff4f6be15b7330f66a9a2c8,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 364 |
+
54fee0f8bc984848c7d80af120befb83dc9bc79dcff4f6be15b7330f66a9a2c8,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 365 |
+
54fee0f8bc984848c7d80af120befb83dc9bc79dcff4f6be15b7330f66a9a2c8,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.600042641162872e-06
|
| 366 |
+
4f8d05647479446b0f85f1266b4c344b1815f621abc8da2dead93d85a3298d91,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 367 |
+
4f8d05647479446b0f85f1266b4c344b1815f621abc8da2dead93d85a3298d91,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 368 |
+
4f8d05647479446b0f85f1266b4c344b1815f621abc8da2dead93d85a3298d91,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 369 |
+
4f8d05647479446b0f85f1266b4c344b1815f621abc8da2dead93d85a3298d91,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.999805241823196e-06
|
| 370 |
+
cdb347a640eab365a60ffb196829230800ab5897246535246f7daff0ac7d5e79,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 371 |
+
cdb347a640eab365a60ffb196829230800ab5897246535246f7daff0ac7d5e79,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 372 |
+
cdb347a640eab365a60ffb196829230800ab5897246535246f7daff0ac7d5e79,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 373 |
+
cdb347a640eab365a60ffb196829230800ab5897246535246f7daff0ac7d5e79,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.699919372797012e-06
|
| 374 |
+
73c0decf1aad751658b64179342fbc7c8daa68fd21a423323218a5402a16264f,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 375 |
+
73c0decf1aad751658b64179342fbc7c8daa68fd21a423323218a5402a16264f,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 376 |
+
73c0decf1aad751658b64179342fbc7c8daa68fd21a423323218a5402a16264f,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 377 |
+
73c0decf1aad751658b64179342fbc7c8daa68fd21a423323218a5402a16264f,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,6.400048732757568e-06
|
| 378 |
+
59de0060f54e97e76078865aff7a1cdb687ebe7f2e675cfb7f60a5c68681a5ba,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 379 |
+
59de0060f54e97e76078865aff7a1cdb687ebe7f2e675cfb7f60a5c68681a5ba,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 380 |
+
59de0060f54e97e76078865aff7a1cdb687ebe7f2e675cfb7f60a5c68681a5ba,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 381 |
+
59de0060f54e97e76078865aff7a1cdb687ebe7f2e675cfb7f60a5c68681a5ba,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.200039595365524e-06
|
| 382 |
+
46ebcfd70bc613dbce3994fd64b0196d408cab68f294db6d630fc92f9e88ddfc,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 383 |
+
46ebcfd70bc613dbce3994fd64b0196d408cab68f294db6d630fc92f9e88ddfc,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 384 |
+
46ebcfd70bc613dbce3994fd64b0196d408cab68f294db6d630fc92f9e88ddfc,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 385 |
+
46ebcfd70bc613dbce3994fd64b0196d408cab68f294db6d630fc92f9e88ddfc,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.400033503770828e-06
|
| 386 |
+
5c8773566c9c879d6b5031361279dc0c23a79dc40f772b479d5ca0c465e5cbac,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 387 |
+
5c8773566c9c879d6b5031361279dc0c23a79dc40f772b479d5ca0c465e5cbac,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 388 |
+
5c8773566c9c879d6b5031361279dc0c23a79dc40f772b479d5ca0c465e5cbac,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 389 |
+
5c8773566c9c879d6b5031361279dc0c23a79dc40f772b479d5ca0c465e5cbac,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,8.599832653999329e-06
|
| 390 |
+
908e8dffd292f7866898dfe1c3cdb76ee0e44f8ed158a7d035e27d050c8902e2,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 391 |
+
908e8dffd292f7866898dfe1c3cdb76ee0e44f8ed158a7d035e27d050c8902e2,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 392 |
+
908e8dffd292f7866898dfe1c3cdb76ee0e44f8ed158a7d035e27d050c8902e2,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 393 |
+
908e8dffd292f7866898dfe1c3cdb76ee0e44f8ed158a7d035e27d050c8902e2,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.400048732757568e-06
|
| 394 |
+
dd617cb98c1c0b9218d4d4804149b4709a31b9d231058c7fb6e3efe8ceb909c8,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 395 |
+
dd617cb98c1c0b9218d4d4804149b4709a31b9d231058c7fb6e3efe8ceb909c8,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 396 |
+
dd617cb98c1c0b9218d4d4804149b4709a31b9d231058c7fb6e3efe8ceb909c8,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 397 |
+
dd617cb98c1c0b9218d4d4804149b4709a31b9d231058c7fb6e3efe8ceb909c8,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.899928510189056e-06
|
| 398 |
+
c71ddf9d811676fb277c76657ed55557d77ff96db45dbd1c38a74ce488babb69,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 399 |
+
c71ddf9d811676fb277c76657ed55557d77ff96db45dbd1c38a74ce488babb69,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 400 |
+
c71ddf9d811676fb277c76657ed55557d77ff96db45dbd1c38a74ce488babb69,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 401 |
+
c71ddf9d811676fb277c76657ed55557d77ff96db45dbd1c38a74ce488babb69,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.200280040502548e-06
|
| 402 |
+
490f948de9d1a503ed1ddc4f31fad62b437a7187defb5ef416a3a604823dbe66,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 403 |
+
490f948de9d1a503ed1ddc4f31fad62b437a7187defb5ef416a3a604823dbe66,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 404 |
+
490f948de9d1a503ed1ddc4f31fad62b437a7187defb5ef416a3a604823dbe66,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 405 |
+
490f948de9d1a503ed1ddc4f31fad62b437a7187defb5ef416a3a604823dbe66,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.200280040502548e-06
|
| 406 |
+
37af6469eb854c4c424648e547f21f49305627d8859f0f47ab0593183f87174d,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 407 |
+
37af6469eb854c4c424648e547f21f49305627d8859f0f47ab0593183f87174d,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 408 |
+
37af6469eb854c4c424648e547f21f49305627d8859f0f47ab0593183f87174d,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 409 |
+
37af6469eb854c4c424648e547f21f49305627d8859f0f47ab0593183f87174d,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.599802196025848e-06
|
| 410 |
+
592cc81f85c79522d0831e2fecda575e5f76b65f57d46330ab63921cfa3fcdf6,3,CERT_1_TO_2,1,0,2,,True,0,0
|
| 411 |
+
592cc81f85c79522d0831e2fecda575e5f76b65f57d46330ab63921cfa3fcdf6,4,CERT_1_TO_2,1,0,2,,True,0,0
|
| 412 |
+
592cc81f85c79522d0831e2fecda575e5f76b65f57d46330ab63921cfa3fcdf6,5,CERT_1_TO_2,1,0,2,,True,0,0
|
| 413 |
+
592cc81f85c79522d0831e2fecda575e5f76b65f57d46330ab63921cfa3fcdf6,6,CERT_1_TO_2,1,0,3,ACTIVE,True,1,3.899913281202316e-06
|
| 414 |
+
4c3097b28df58eb5dee18108de39082e2380c8776883f9cdb836e60b6cb05cf7,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 415 |
+
4c3097b28df58eb5dee18108de39082e2380c8776883f9cdb836e60b6cb05cf7,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 416 |
+
4c3097b28df58eb5dee18108de39082e2380c8776883f9cdb836e60b6cb05cf7,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 417 |
+
4c3097b28df58eb5dee18108de39082e2380c8776883f9cdb836e60b6cb05cf7,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.899928510189056e-06
|
| 418 |
+
45fa8fcd4626f600b9274b0dafdad42aef9abf23786096bfa5ddba7f8aad9a26,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 419 |
+
45fa8fcd4626f600b9274b0dafdad42aef9abf23786096bfa5ddba7f8aad9a26,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 420 |
+
45fa8fcd4626f600b9274b0dafdad42aef9abf23786096bfa5ddba7f8aad9a26,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 421 |
+
45fa8fcd4626f600b9274b0dafdad42aef9abf23786096bfa5ddba7f8aad9a26,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.699919372797012e-06
|
| 422 |
+
8b0ea00d5889f55f314602dd46d665aa52bb5ba148530066d522e8180b008375,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 423 |
+
8b0ea00d5889f55f314602dd46d665aa52bb5ba148530066d522e8180b008375,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 424 |
+
8b0ea00d5889f55f314602dd46d665aa52bb5ba148530066d522e8180b008375,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 425 |
+
8b0ea00d5889f55f314602dd46d665aa52bb5ba148530066d522e8180b008375,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.299916326999664e-06
|
| 426 |
+
43970d8c5e1b70636e7a4cb70919c4c44ff23661a628fdf590ddef34cc13cfd9,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 427 |
+
43970d8c5e1b70636e7a4cb70919c4c44ff23661a628fdf590ddef34cc13cfd9,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 428 |
+
43970d8c5e1b70636e7a4cb70919c4c44ff23661a628fdf590ddef34cc13cfd9,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 429 |
+
43970d8c5e1b70636e7a4cb70919c4c44ff23661a628fdf590ddef34cc13cfd9,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.500150680541992e-06
|
| 430 |
+
b2a5eb488b3a17d412e386f2b1c1c7315a890948a8506b667329d3aedc6e0bb0,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 431 |
+
b2a5eb488b3a17d412e386f2b1c1c7315a890948a8506b667329d3aedc6e0bb0,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 432 |
+
b2a5eb488b3a17d412e386f2b1c1c7315a890948a8506b667329d3aedc6e0bb0,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 433 |
+
b2a5eb488b3a17d412e386f2b1c1c7315a890948a8506b667329d3aedc6e0bb0,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.299931555986404e-06
|
| 434 |
+
da6686271f876b7b5cf4348d2e52301440f21f1a7061792aa8eeb51c2dc319d6,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 435 |
+
da6686271f876b7b5cf4348d2e52301440f21f1a7061792aa8eeb51c2dc319d6,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 436 |
+
da6686271f876b7b5cf4348d2e52301440f21f1a7061792aa8eeb51c2dc319d6,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 437 |
+
da6686271f876b7b5cf4348d2e52301440f21f1a7061792aa8eeb51c2dc319d6,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,5.899928510189056e-06
|
| 438 |
+
ea5c5c547e8ad94c534b5d0fd1292d8062052b39fe6c5709b3bdb3f3470fca93,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 439 |
+
ea5c5c547e8ad94c534b5d0fd1292d8062052b39fe6c5709b3bdb3f3470fca93,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 440 |
+
ea5c5c547e8ad94c534b5d0fd1292d8062052b39fe6c5709b3bdb3f3470fca93,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 441 |
+
ea5c5c547e8ad94c534b5d0fd1292d8062052b39fe6c5709b3bdb3f3470fca93,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.599802196025848e-06
|
| 442 |
+
5d2f33817dd25ab2e48b9c4a2d36b44b211d3c1719484665bf14139d3ca8c893,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 443 |
+
5d2f33817dd25ab2e48b9c4a2d36b44b211d3c1719484665bf14139d3ca8c893,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 444 |
+
5d2f33817dd25ab2e48b9c4a2d36b44b211d3c1719484665bf14139d3ca8c893,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 445 |
+
5d2f33817dd25ab2e48b9c4a2d36b44b211d3c1719484665bf14139d3ca8c893,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,3.700144588947296e-06
|
| 446 |
+
6cbae0878fe19cd3eb5a7b565a7a7bec5646f91cdd7d02914d37e036568bf693,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 447 |
+
6cbae0878fe19cd3eb5a7b565a7a7bec5646f91cdd7d02914d37e036568bf693,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 448 |
+
6cbae0878fe19cd3eb5a7b565a7a7bec5646f91cdd7d02914d37e036568bf693,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 449 |
+
6cbae0878fe19cd3eb5a7b565a7a7bec5646f91cdd7d02914d37e036568bf693,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.00003045797348e-06
|
| 450 |
+
c24aa242bd7c084e8cf07ae9948189cee402c08bafe3f0f84a7c0ac33ef462a1,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 451 |
+
c24aa242bd7c084e8cf07ae9948189cee402c08bafe3f0f84a7c0ac33ef462a1,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 452 |
+
c24aa242bd7c084e8cf07ae9948189cee402c08bafe3f0f84a7c0ac33ef462a1,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 453 |
+
c24aa242bd7c084e8cf07ae9948189cee402c08bafe3f0f84a7c0ac33ef462a1,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,6.400048732757568e-06
|
| 454 |
+
a3b1804ffcf480de5f2331e3a270916a60994af01d07a75b6187d9bc88d13229,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 455 |
+
a3b1804ffcf480de5f2331e3a270916a60994af01d07a75b6187d9bc88d13229,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 456 |
+
a3b1804ffcf480de5f2331e3a270916a60994af01d07a75b6187d9bc88d13229,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 457 |
+
a3b1804ffcf480de5f2331e3a270916a60994af01d07a75b6187d9bc88d13229,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.600283086299896e-06
|
| 458 |
+
e0bd53aff0d6693304881d5129ae28690d2dff6c813474a182d931fb3ef3cfbd,3,CERT_1_TO_2,2,0,1,,True,0,0
|
| 459 |
+
e0bd53aff0d6693304881d5129ae28690d2dff6c813474a182d931fb3ef3cfbd,4,CERT_1_TO_2,2,0,1,,True,0,0
|
| 460 |
+
e0bd53aff0d6693304881d5129ae28690d2dff6c813474a182d931fb3ef3cfbd,5,CERT_1_TO_2,2,0,1,,True,0,0
|
| 461 |
+
e0bd53aff0d6693304881d5129ae28690d2dff6c813474a182d931fb3ef3cfbd,6,CERT_1_TO_2,2,0,1,IDENTITY,True,1,4.699919372797012e-06
|
| 462 |
+
24edd28f40e78f36e68eb4e982ddc44576788b7759d0bc8ab574e8b0a066e98a,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 463 |
+
24edd28f40e78f36e68eb4e982ddc44576788b7759d0bc8ab574e8b0a066e98a,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 464 |
+
24edd28f40e78f36e68eb4e982ddc44576788b7759d0bc8ab574e8b0a066e98a,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 465 |
+
24edd28f40e78f36e68eb4e982ddc44576788b7759d0bc8ab574e8b0a066e98a,6,CERT_1_TO_2,1,0,1,IDENTITY,True,3,9.899958968162537e-06
|
| 466 |
+
e72bb0a112ccca955fc6b5f93975f5d96968d16751b29f449c7ebe563ae66b03,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 467 |
+
e72bb0a112ccca955fc6b5f93975f5d96968d16751b29f449c7ebe563ae66b03,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 468 |
+
e72bb0a112ccca955fc6b5f93975f5d96968d16751b29f449c7ebe563ae66b03,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 469 |
+
e72bb0a112ccca955fc6b5f93975f5d96968d16751b29f449c7ebe563ae66b03,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,6.4997002482414246e-06
|
| 470 |
+
6f44f7acd97e6be654d56f69a6e24addc50821ff94457fd1af9fadb3a95e30bd,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 471 |
+
6f44f7acd97e6be654d56f69a6e24addc50821ff94457fd1af9fadb3a95e30bd,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 472 |
+
6f44f7acd97e6be654d56f69a6e24addc50821ff94457fd1af9fadb3a95e30bd,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 473 |
+
6f44f7acd97e6be654d56f69a6e24addc50821ff94457fd1af9fadb3a95e30bd,6,CERT_1_TO_2,1,0,1,IDENTITY,True,2,6.100162863731384e-06
|
| 474 |
+
18464b7498c06e4be0064a2d12be6673345827ea1736d79bb15aed46835ac60b,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 475 |
+
18464b7498c06e4be0064a2d12be6673345827ea1736d79bb15aed46835ac60b,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 476 |
+
18464b7498c06e4be0064a2d12be6673345827ea1736d79bb15aed46835ac60b,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 477 |
+
18464b7498c06e4be0064a2d12be6673345827ea1736d79bb15aed46835ac60b,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.90015372633934e-06
|
| 478 |
+
652e636f3e7a008b2f6437ac35d17172ec2af0a76a5951542856290b8f852079,3,CERT_1_TO_2,1,0,1,,True,0,0
|
| 479 |
+
652e636f3e7a008b2f6437ac35d17172ec2af0a76a5951542856290b8f852079,4,CERT_1_TO_2,1,0,1,,True,0,0
|
| 480 |
+
652e636f3e7a008b2f6437ac35d17172ec2af0a76a5951542856290b8f852079,5,CERT_1_TO_2,1,0,1,,True,0,0
|
| 481 |
+
652e636f3e7a008b2f6437ac35d17172ec2af0a76a5951542856290b8f852079,6,CERT_1_TO_2,1,0,1,IDENTITY,True,1,4.00003045797348e-06
|
data/families.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/model_contract.json
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"M1": {
|
| 3 |
+
"repo_id": "Qwen/Qwen3-8B",
|
| 4 |
+
"revision": "b968826d9c46dd6066d109eabc6255188de91218",
|
| 5 |
+
"dtype": "bfloat16",
|
| 6 |
+
"attention": "sdpa",
|
| 7 |
+
"batch_size": 8,
|
| 8 |
+
"generation_config": {
|
| 9 |
+
"max_length": 20,
|
| 10 |
+
"max_new_tokens": null,
|
| 11 |
+
"min_length": 0,
|
| 12 |
+
"min_new_tokens": 0,
|
| 13 |
+
"early_stopping": false,
|
| 14 |
+
"max_time": null,
|
| 15 |
+
"stop_strings": null,
|
| 16 |
+
"do_sample": true,
|
| 17 |
+
"num_beams": 1,
|
| 18 |
+
"use_cache": true,
|
| 19 |
+
"cache_implementation": null,
|
| 20 |
+
"cache_config": null,
|
| 21 |
+
"return_legacy_cache": null,
|
| 22 |
+
"prefill_chunk_size": null,
|
| 23 |
+
"temperature": 0.6,
|
| 24 |
+
"top_k": 20,
|
| 25 |
+
"top_p": 0.95,
|
| 26 |
+
"min_p": 0.0,
|
| 27 |
+
"typical_p": 1.0,
|
| 28 |
+
"epsilon_cutoff": 0.0,
|
| 29 |
+
"eta_cutoff": 0.0,
|
| 30 |
+
"repetition_penalty": 1.0,
|
| 31 |
+
"encoder_repetition_penalty": 1.0,
|
| 32 |
+
"length_penalty": 1.0,
|
| 33 |
+
"no_repeat_ngram_size": 0,
|
| 34 |
+
"bad_words_ids": null,
|
| 35 |
+
"renormalize_logits": false,
|
| 36 |
+
"forced_bos_token_id": null,
|
| 37 |
+
"forced_eos_token_id": null,
|
| 38 |
+
"remove_invalid_values": false,
|
| 39 |
+
"exponential_decay_length_penalty": null,
|
| 40 |
+
"suppress_tokens": null,
|
| 41 |
+
"begin_suppress_tokens": null,
|
| 42 |
+
"sequence_bias": null,
|
| 43 |
+
"token_healing": false,
|
| 44 |
+
"guidance_scale": null,
|
| 45 |
+
"watermarking_config": null,
|
| 46 |
+
"num_return_sequences": 1,
|
| 47 |
+
"output_attentions": false,
|
| 48 |
+
"output_hidden_states": false,
|
| 49 |
+
"output_scores": false,
|
| 50 |
+
"output_logits": null,
|
| 51 |
+
"return_dict_in_generate": false,
|
| 52 |
+
"pad_token_id": 151643,
|
| 53 |
+
"bos_token_id": 151643,
|
| 54 |
+
"eos_token_id": [
|
| 55 |
+
151645,
|
| 56 |
+
151643
|
| 57 |
+
],
|
| 58 |
+
"encoder_no_repeat_ngram_size": 0,
|
| 59 |
+
"decoder_start_token_id": null,
|
| 60 |
+
"is_assistant": false,
|
| 61 |
+
"num_assistant_tokens": 20,
|
| 62 |
+
"num_assistant_tokens_schedule": "constant",
|
| 63 |
+
"assistant_confidence_threshold": 0.4,
|
| 64 |
+
"prompt_lookup_num_tokens": null,
|
| 65 |
+
"max_matching_ngram_size": null,
|
| 66 |
+
"assistant_early_exit": null,
|
| 67 |
+
"assistant_lookbehind": 10,
|
| 68 |
+
"target_lookbehind": 10,
|
| 69 |
+
"disable_compile": false,
|
| 70 |
+
"low_memory": null,
|
| 71 |
+
"penalty_alpha": null,
|
| 72 |
+
"dola_layers": null,
|
| 73 |
+
"diversity_penalty": 0.0,
|
| 74 |
+
"num_beam_groups": 1,
|
| 75 |
+
"constraints": null,
|
| 76 |
+
"force_words_ids": null,
|
| 77 |
+
"_from_model_config": false,
|
| 78 |
+
"transformers_version": "4.57.1"
|
| 79 |
+
},
|
| 80 |
+
"endpoint_profile": "PILOT02_THINK_BOUNDARY",
|
| 81 |
+
"eos_ids": [
|
| 82 |
+
151645,
|
| 83 |
+
151643
|
| 84 |
+
],
|
| 85 |
+
"eos_strings": [
|
| 86 |
+
"<|im_end|>",
|
| 87 |
+
"<|endoftext|>"
|
| 88 |
+
],
|
| 89 |
+
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
|
| 90 |
+
"native_config_class": "Qwen3Config",
|
| 91 |
+
"trust_remote_code": false
|
| 92 |
+
},
|
| 93 |
+
"M2": {
|
| 94 |
+
"repo_id": "allenai/Olmo-3-7B-Think",
|
| 95 |
+
"revision": "d97e442d7cc678210054dbcc9b440894d62c89a4",
|
| 96 |
+
"dtype": "bfloat16",
|
| 97 |
+
"attention": "sdpa",
|
| 98 |
+
"batch_size": 4,
|
| 99 |
+
"generation_config": {
|
| 100 |
+
"max_length": 20,
|
| 101 |
+
"max_new_tokens": 32768,
|
| 102 |
+
"min_length": 0,
|
| 103 |
+
"min_new_tokens": 0,
|
| 104 |
+
"early_stopping": false,
|
| 105 |
+
"max_time": null,
|
| 106 |
+
"stop_strings": null,
|
| 107 |
+
"do_sample": true,
|
| 108 |
+
"num_beams": 1,
|
| 109 |
+
"use_cache": true,
|
| 110 |
+
"cache_implementation": null,
|
| 111 |
+
"cache_config": null,
|
| 112 |
+
"return_legacy_cache": null,
|
| 113 |
+
"prefill_chunk_size": null,
|
| 114 |
+
"temperature": 0.6,
|
| 115 |
+
"top_k": 0,
|
| 116 |
+
"top_p": 0.95,
|
| 117 |
+
"min_p": 0.0,
|
| 118 |
+
"typical_p": 1.0,
|
| 119 |
+
"epsilon_cutoff": 0.0,
|
| 120 |
+
"eta_cutoff": 0.0,
|
| 121 |
+
"repetition_penalty": 1.0,
|
| 122 |
+
"encoder_repetition_penalty": 1.0,
|
| 123 |
+
"length_penalty": 1.0,
|
| 124 |
+
"no_repeat_ngram_size": 0,
|
| 125 |
+
"bad_words_ids": null,
|
| 126 |
+
"renormalize_logits": false,
|
| 127 |
+
"forced_bos_token_id": null,
|
| 128 |
+
"forced_eos_token_id": null,
|
| 129 |
+
"remove_invalid_values": false,
|
| 130 |
+
"exponential_decay_length_penalty": null,
|
| 131 |
+
"suppress_tokens": null,
|
| 132 |
+
"begin_suppress_tokens": null,
|
| 133 |
+
"sequence_bias": null,
|
| 134 |
+
"token_healing": false,
|
| 135 |
+
"guidance_scale": null,
|
| 136 |
+
"watermarking_config": null,
|
| 137 |
+
"num_return_sequences": 1,
|
| 138 |
+
"output_attentions": false,
|
| 139 |
+
"output_hidden_states": false,
|
| 140 |
+
"output_scores": false,
|
| 141 |
+
"output_logits": null,
|
| 142 |
+
"return_dict_in_generate": false,
|
| 143 |
+
"pad_token_id": 100277,
|
| 144 |
+
"bos_token_id": null,
|
| 145 |
+
"eos_token_id": [
|
| 146 |
+
100265,
|
| 147 |
+
100257
|
| 148 |
+
],
|
| 149 |
+
"encoder_no_repeat_ngram_size": 0,
|
| 150 |
+
"decoder_start_token_id": null,
|
| 151 |
+
"is_assistant": false,
|
| 152 |
+
"num_assistant_tokens": 20,
|
| 153 |
+
"num_assistant_tokens_schedule": "constant",
|
| 154 |
+
"assistant_confidence_threshold": 0.4,
|
| 155 |
+
"prompt_lookup_num_tokens": null,
|
| 156 |
+
"max_matching_ngram_size": null,
|
| 157 |
+
"assistant_early_exit": null,
|
| 158 |
+
"assistant_lookbehind": 10,
|
| 159 |
+
"target_lookbehind": 10,
|
| 160 |
+
"disable_compile": false,
|
| 161 |
+
"low_memory": null,
|
| 162 |
+
"penalty_alpha": null,
|
| 163 |
+
"dola_layers": null,
|
| 164 |
+
"diversity_penalty": 0.0,
|
| 165 |
+
"num_beam_groups": 1,
|
| 166 |
+
"constraints": null,
|
| 167 |
+
"force_words_ids": null,
|
| 168 |
+
"_from_model_config": true,
|
| 169 |
+
"transformers_version": "4.57.1"
|
| 170 |
+
},
|
| 171 |
+
"endpoint_profile": "PILOT02_THINK_BOUNDARY",
|
| 172 |
+
"eos_ids": [
|
| 173 |
+
100265,
|
| 174 |
+
100257
|
| 175 |
+
],
|
| 176 |
+
"eos_strings": [
|
| 177 |
+
"<|im_end|>",
|
| 178 |
+
"<|endoftext|>"
|
| 179 |
+
],
|
| 180 |
+
"chat_template": "{% set has_system = messages|selectattr('role', 'equalto', 'system')|list|length > 0 %}{% if not has_system %}{{ '<|im_start|>system\nYou are OLMo, a helpful function-calling AI assistant built by Ai2. Your date cutoff is November 2024, and your model weights are available at https://huggingface.co/allenai. You do not currently have access to any functions. <functions></functions><|im_end|>\n' }}{% endif %}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|im_start|>system\n' + message['content'] }}{% if message.get('functions', none) is not none %}{{ ' <functions>' + message['functions'] + '</functions><|im_end|>\n' }}{% else %}{{ ' You do not currently have access to any functions. <functions></functions><|im_end|>\n' }}{% endif %}{% elif message['role'] == 'user' %}{% if message.get('functions', none) is not none %}{{ '<|im_start|>user\n' + message['content'] + '\n' + '<functions>' + message['functions'] + '</functions><|im_end|>\n' }}{% else %}{{ '<|im_start|>user\n' + message['content'] + '<|im_end|>\n' }}{% endif %}{% elif message['role'] == 'assistant' %}{{ '<|im_start|>assistant\n' }}{% if message.get('content', none) is not none %}{{ message['content'] }}{% endif %}{% if message.get('function_calls', none) is not none %}{{ '<function_calls>' + message['function_calls'] + '</function_calls>' }}{% endif %}{% if not loop.last %}{{ '<|im_end|>' + '\n' }}{% else %}{{ eos_token }}{% endif %}{% elif message['role'] == 'environment' %}{{ '<|im_start|>environment\n' + message['content'] + '<|im_end|>\n' }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|im_start|>assistant\n<think>' }}{% endif %}{% endfor %}",
|
| 181 |
+
"native_config_class": "Olmo3Config",
|
| 182 |
+
"trust_remote_code": false
|
| 183 |
+
}
|
| 184 |
+
}
|
data/native1008.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/selected_programs.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/synthesis_instruction.txt
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Infer a cascade of string-replacement rules that maps every input to its paired output.
|
| 2 |
+
Apply each rule to every string in order, using Python str.replace semantics.
|
| 3 |
+
Use only characters from abcdefghijkuvwxyz.
|
| 4 |
+
The first argument must contain 1 to 3 characters; the second may contain 0 to 3 characters.
|
| 5 |
+
Use at most 5 rules. Any functionally correct cascade is acceptable.
|
| 6 |
+
You may reason before answering. Put your final answer in one fenced block labelled python.
|
| 7 |
+
Inside that block write a Python list whose elements are strings. Each string must be a rule written as replace('a','b'), with two quoted arguments satisfying the limits above.
|
| 8 |
+
For example, the serialization of one rule is:
|
| 9 |
+
```python
|
| 10 |
+
["replace('a','b')"]
|
| 11 |
+
```
|
| 12 |
+
This example shows notation only, not a proposed solution. Keep the order of your rules. Arrow notation, bare character pairs, and a list of output strings are not the requested rule-list format.
|
| 13 |
+
Input-output pairs:
|
data/trajectories.jsonl.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6d9c9af078d6ec25bb8f1e95894ed07dbd879c2e3d77b7dc5afe0081cf57ed7d
|
| 3 |
+
size 26517321
|
docs/asset-manifest.csv
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
path,bytes,sha256,origin,scope
|
| 2 |
+
code/canonical_readouts_v1.py,4513,793af7b98682cb57b5efc47b5b534e86ef90725f7c3269b4fc098d76778126a6,frozen study artifact,saved_output_count_panel_evaluation
|
| 3 |
+
code/family_statistics.py,854,2dd50ccb2d0e3819efa79e075573d8ce799ceb18011a9ba6229e0bc4bd095a83,frozen study artifact,saved_output_count_panel_evaluation
|
| 4 |
+
code/frozen/PBEBench_functional_correctness.py,24616,db8ab01ef70be0c4ad34610045be4c6f6adad03977921efefc16260382901322,frozen study artifact,archived_scoring_and_analysis_source
|
| 5 |
+
code/frozen/PBEBench_primitives.py,4390,9d68d96f6a86413f1f5849a4f589bc7ea009e5987555d7f14308bb851e92c043,frozen study artifact,archived_scoring_and_analysis_source
|
| 6 |
+
code/frozen/PBEBench_utils.py,215,418c27074bdcad4a3a3a1ff3a532b0ee7e9f46a2f5d41cf6da7908e5b8673b90,frozen study artifact,archived_scoring_and_analysis_source
|
| 7 |
+
code/frozen/analyze_panel.py,8111,b6be13e25962d341bc24286c9d42b688487de49c62509598add793baa66d57aa,frozen study artifact,archived_scoring_and_analysis_source
|
| 8 |
+
code/frozen/functional_score.py,1576,b1fffcb1ec769693cd6100423d9ba3030ce1976674e62a1c9e0cb2cf6ccee904,frozen study artifact,archived_scoring_and_analysis_source
|
| 9 |
+
code/native_score.py,8581,3174025e5904cbd7a5f5f31385facd6a2fb7e4c64d4f857a3e1381fe792c3425,frozen study artifact,saved_output_count_panel_evaluation
|
| 10 |
+
data/analysis_contract.json,2169,4a6991cad6350c7c6fb29c5b0a9d8f0929f3e4b0149f9c05b720970b801ee1cd,frozen study artifact,saved_output_count_panel_evaluation
|
| 11 |
+
data/base_variants.jsonl,965314,9cd4c40b7a1468a699c40a6aacda61e73e507868d2b9cf02ad98cbbda89006d1,frozen study artifact,960_arm_certificates | 480_paired_base_variants
|
| 12 |
+
data/broader_inputs.jsonl,533575,520948e896fa37c8d0389b035ad84afc9a741a20c9e3ecf8173f49771b421fe5,frozen study artifact,7680_reference_labels | 3840_sequential_two_rule_activation_checks
|
| 13 |
+
data/certificate_records.jsonl,648598,dab7802e8005f615c3b0f3c7dac92d34ffe547b2f9b8f55e1fcd101c4de5b850,frozen study artifact,960_arm_certificates | 480_paired_base_variants
|
| 14 |
+
data/construction_audit.csv,49640,de6d2a61a2b518f8d04b0351b290a19d684c16a704d9aa8eec6cb2bf5bbd9ff9,frozen study artifact,960_arm_certificates | 480_paired_base_variants
|
| 15 |
+
data/families.jsonl,309171,c6f7814d9085f597b81d310115cf42f03efe95f7c7cdec48cacdffb69b5198b7,frozen study artifact,120_family_task_and_witness_specifications
|
| 16 |
+
data/model_contract.json,11284,b33554033c16b56605bd384a885001d052b3a0e346487ade40b4ff9ff354466a,frozen study artifact,pinned_inference_configuration_or_prompt
|
| 17 |
+
data/native1008.csv,387271,76faa6b2f0f35874a840c11e1ef6cd3180d1665a856dc64e97293492a92a0343,frozen study artifact,1008_task_certificate_audit
|
| 18 |
+
data/selected_programs.csv,656183,94ca7ea378fce2954520f6641505440257bbb1529257f36ea66fc37dd4892e67,frozen study artifact,saved_output_count_panel_evaluation
|
| 19 |
+
data/synthesis_instruction.txt,938,4fb7efda45f0beb0978934612eeeca36e73da3d87f4977c17dd5f4d386318f11,frozen study artifact,pinned_inference_configuration_or_prompt
|
| 20 |
+
data/trajectories.jsonl.gz,26517321,6d9c9af078d6ec25bb8f1e95894ed07dbd879c2e3d77b7dc5afe0081cf57ed7d,frozen study artifact,saved_output_count_panel_evaluation
|
| 21 |
+
fixtures/end_to_end.json,5396,de641ad2106e9b1fdc9e64214f0938a20e02f230ba4db88291e9c6a60dda59cf,frozen study artifact,one_rule_enumeration | two_rule_witness | saved_output_selection | displayed_and_new_input_execution
|
| 22 |
+
provenance/ANALYSIS_HISTORY.json,4314,9b8872616c8a65248574431a62ddf41338651b20af31edf1eb5689d517cf55ff,frozen study artifact,historical_analysis_plan_transcription
|
| 23 |
+
provenance/NORMALIZATION.json,1289,10c7e5e51c9b62b4c59f76bc90c2ece4813ababcc687394757f07359279e87ce,frozen study artifact,versioned_provenance_record
|
| 24 |
+
replay_fixture.py,5708,4a35106cc2c0d8e84222c037ec56b6157b489fc91d2d19fd12e5a6c686216eb8,frozen study artifact,one_rule_enumeration | two_rule_witness | saved_output_selection | displayed_and_new_input_execution
|
| 25 |
+
reproduce.py,16662,91998c900c54b85fd13551014abc16038de947c7741fea192309bf26edce2fb5,frozen study artifact,saved_output_count_panel_evaluation
|
| 26 |
+
requirements.txt,15,99f43573522565d7c480efd1f954892948033915df6a4e685327e995d3396eb6,frozen study artifact,instructions_or_version_history
|
| 27 |
+
results/BROADER_PANEL_READABLE.csv,2275,b2ed0a1e44acebb2c4c85e826be278a529bc9e72878cb8df6e7b2305d419bfc9,frozen study artifact,saved_output_count_panel_evaluation
|
| 28 |
+
results/PRIMARY_PANEL_READABLE.csv,1366,16868778301c28cdb7441572059229e3610903cd426bad122cad2eeb33c09102,frozen study artifact,saved_output_count_panel_evaluation
|
| 29 |
+
results/base_only_solution_counts.csv,43845,868346768fdb71fa0c80c52beee3c09ab685f14fe70f968f2c302e6bed73f144,frozen study artifact,960_arm_certificates | 480_paired_base_variants
|
| 30 |
+
results/broader_original_immutable.json,162066,db2c1c28672538d8144e8905bde9f885065d58fd4b20c7c302218372e8b2e30e,frozen study artifact,recorded_original_broader_input_summary
|
| 31 |
+
results/family_paired_differences.csv,322677,10aacaa32c80bef20442cb7d1aadd97fe1cccbb2131dba0d85ca525bb787b9f3,frozen study artifact,saved_output_count_panel_evaluation
|
| 32 |
+
results/paired_effects.csv,64158,5085090c3147e28f00184546201d6b1d7e06b7bcc70b42a9e468625b266f4b90,frozen study artifact,saved_output_count_panel_evaluation
|
| 33 |
+
results/per_family_metrics.csv,434378,18e2ebd712fd2cebe6f7e766acc2d67707d29eebbf5a28ae00a0ed34aee008d6,frozen study artifact,saved_output_count_panel_evaluation
|
| 34 |
+
verify_manifest.py,1466,ca7cd1faea2195c447c5d3596b125fd8bc93ff9a2db050403e71602c3f27ffb9,frozen study artifact,current_file_hashes_and_sizes
|
| 35 |
+
worked_case.json,9537,6c80328e4c960f1b3a0d26d499cc7ef0d2e091203f48a18b6bc39988fd4851a5,frozen study artifact,shortcut_return_and_identity_constraint_certificate
|
| 36 |
+
licenses/PBEBench-MIT.txt,1068,7ee6673ea31294494031ee577102e866bcf6438280142f822f1497cd153543e1,PBEBench pinned upstream license,
|
| 37 |
+
docs/figures/construction.pdf,83247,1224ffa9a3dfa361a7661a9fea3763250414cf84380e6cb7dc4e0c211ac9396f,manuscript Figure 1,
|
| 38 |
+
docs/figures/construction.png,140513,73c9776e67c7fdb2524682b4a4759aa2a319843c7a667b2429cad369799291e1,manuscript Figure 1 raster,
|
| 39 |
+
docs/reproduction_report.json,2525,4aaac315cc92a9c0b51198d8fe7285243940155f0b61913d4b8951573a47711d,CPU verification of frozen artifact,
|
docs/companion.json
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"article": {
|
| 3 |
+
"version": "1.0",
|
| 4 |
+
"published": "2026-11-20",
|
| 5 |
+
"updated": "2026-11-20",
|
| 6 |
+
"updateNote": {
|
| 7 |
+
"en": "Companion to arXiv v1 and the v1.0.0 CPU artifact.",
|
| 8 |
+
"zh": "对应 arXiv v1 与 v1.0.0 CPU 复现归档。"
|
| 9 |
+
}
|
| 10 |
+
},
|
| 11 |
+
"locales": {
|
| 12 |
+
"en": {
|
| 13 |
+
"title": "When does an example force a second rule?",
|
| 14 |
+
"summary": "A model can explain every demonstration with a shortcut. One carefully chosen example lets us certify when that shortcut disappears, then measure what the model does on the task and on new strings."
|
| 15 |
+
},
|
| 16 |
+
"zh": {
|
| 17 |
+
"title": "什么样的样例会迫使程序使用第二条规则?",
|
| 18 |
+
"summary": "模型可能靠一条捷径解释题面上的所有样例。这项研究只替换一个样例,严格核验第二条规则何时成为必要条件,再分别检查模型对题面和新字符串的判断。"
|
| 19 |
+
}
|
| 20 |
+
},
|
| 21 |
+
"artifact": {
|
| 22 |
+
"version": "v1.0.0",
|
| 23 |
+
"url": "https://github.com/dqswordman/program-structure-certificates/releases/tag/v1.0.0",
|
| 24 |
+
"space": "https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates",
|
| 25 |
+
"guide": "https://github.com/dqswordman/program-structure-certificates/blob/main/docs/reproduction.md",
|
| 26 |
+
"description": {
|
| 27 |
+
"en": "Explore a certified task pair or replay 1,920 saved model trajectories on CPU from the versioned archive.",
|
| 28 |
+
"zh": "在线比较一对经过认证的任务,或从版本归档在 CPU 上重放 1,920 条模型轨迹。"
|
| 29 |
+
}
|
| 30 |
+
},
|
| 31 |
+
"metrics": [
|
| 32 |
+
{
|
| 33 |
+
"value": "120",
|
| 34 |
+
"label": {
|
| 35 |
+
"en": "Reference program families",
|
| 36 |
+
"zh": "参考程序家族"
|
| 37 |
+
}
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"value": "960",
|
| 41 |
+
"label": {
|
| 42 |
+
"en": "Task certificates checked",
|
| 43 |
+
"zh": "经核验的任务证书"
|
| 44 |
+
}
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"value": "1,920",
|
| 48 |
+
"label": {
|
| 49 |
+
"en": "Archived model trajectories",
|
| 50 |
+
"zh": "归档的模型轨迹"
|
| 51 |
+
}
|
| 52 |
+
}
|
| 53 |
+
],
|
| 54 |
+
"sections": [
|
| 55 |
+
{
|
| 56 |
+
"id": "worked-example",
|
| 57 |
+
"title": {
|
| 58 |
+
"en": "Change one example, keep the reference program",
|
| 59 |
+
"zh": "同一个参考程序,只换一道样例"
|
| 60 |
+
},
|
| 61 |
+
"paragraphs": {
|
| 62 |
+
"en": [
|
| 63 |
+
"Imagine inferring a program from six input–output pairs. Five pairs are shared. In the control, the last pair maps iiiexa to iiiazy. The single replacement exa → azy fits all six. The same observations also admit a two-step explanation: replace ex with azw, then wa with y.",
|
| 64 |
+
"In the matched separation task the final pair becomes wbawex → wbayzw. The shortcut leaves wbawex unchanged. The two-step program produces wbawazw after its first rule and wbayzw after its second. All other demonstrations stay fixed. The figure traces both explanations on this archived task."
|
| 65 |
+
],
|
| 66 |
+
"zh": [
|
| 67 |
+
"试着从六组输入输出中推断程序。两道题共享前五组。对照题的最后一组是 iiiexa → iiiazy,用一条 exa → azy 就能解释六组样例。另一个解释有两步:先把 ex 换成 azw,再把 wa 换成 y。",
|
| 68 |
+
"在配对的分离题中,把最后一组换成 wbawex → wbayzw。单规则捷径会原样保留输入;两步程序先得到 wbawazw,再得到 wbayzw。其余五组样例全部相同。下图给出了这对归档任务的完整构造。"
|
| 69 |
+
]
|
| 70 |
+
},
|
| 71 |
+
"blocks": [
|
| 72 |
+
{
|
| 73 |
+
"type": "figure",
|
| 74 |
+
"file": "assets/reports/program-structure-certificates/construction.png",
|
| 75 |
+
"width": 1200,
|
| 76 |
+
"height": 900,
|
| 77 |
+
"alt": {
|
| 78 |
+
"en": "Worked example comparing the control and separation demonstrations and the one- and two-rule programs.",
|
| 79 |
+
"zh": "图中比较对照与分离样例,以及单规则捷径和双规则参考程序。"
|
| 80 |
+
},
|
| 81 |
+
"caption": {
|
| 82 |
+
"en": "Figure 1. Both tasks use the same two-rule reference. The final demonstration determines whether a one-rule solution remains possible.",
|
| 83 |
+
"zh": "图 1。两道题共用同一个双规则参考程序;最后一组样例决定单规则解是否仍然存在。"
|
| 84 |
+
},
|
| 85 |
+
"source": "https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/fixtures/end_to_end.json"
|
| 86 |
+
}
|
| 87 |
+
]
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"id": "certification",
|
| 91 |
+
"title": {
|
| 92 |
+
"en": "What the certificate proves",
|
| 93 |
+
"zh": "证书究竟证明了什么"
|
| 94 |
+
},
|
| 95 |
+
"paragraphs": {
|
| 96 |
+
"en": [
|
| 97 |
+
"Here a rule replaces every non-overlapping occurrence of a literal string. Rules run in sequence. The study fixes a finite alphabet, a left-hand side of one to three characters and a right-hand side of up to three. A changed output rules out the empty program. The verifier enumerates the legal one-rule explanations and executes a two-rule witness.",
|
| 98 |
+
"For this example one single-rule program survives in the control and none survives in separation. The successful witness gives an upper bound of two, so the minimum lengths are exactly one and two. Certification uses the displayed pairs before inference. This statement depends on the chosen language and rule lengths."
|
| 99 |
+
],
|
| 100 |
+
"zh": [
|
| 101 |
+
"每条规则替换字符串中所有互不重叠的字面匹配,多条规则按顺序执行。研究固定有限字母表,左侧长度为一至三个字符,右侧长度最多三个字符。一组发生变化的输出即可排除空程序。验证器随后枚举全部合法单规则解释,并实际执行一个双规则解。",
|
| 102 |
+
"在图中,对照题恰好保留一个单规则解,分离题的单规则解数为零。双规则程序在两题均通过,因此最短长度分别精确为一与二。认证在模型推理之前完成,结论的范围由这套替换语言及规则长度界定。"
|
| 103 |
+
]
|
| 104 |
+
},
|
| 105 |
+
"blocks": []
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"id": "results",
|
| 109 |
+
"title": {
|
| 110 |
+
"en": "Solving the prompt and passing the hidden pair",
|
| 111 |
+
"zh": "题面完成率和隐藏配对的成功率"
|
| 112 |
+
},
|
| 113 |
+
"paragraphs": {
|
| 114 |
+
"en": [
|
| 115 |
+
"Each of the 120 reference families supplies matched control and separation tasks with three to six shared examples, plus one arm-specific example. The panel contains 1,440 new trajectories and 480 previously saved base-five trajectories for Qwen3-8B and Olmo-3-7B-Think. The certified comparison uses 119 families at bases three and four and all 120 at bases five and six.",
|
| 116 |
+
"With five shared examples, Qwen completes 67 of 120 control tasks and 12 separation tasks. Olmo completes 52 and 15. In both models, success on a separate witness pair hidden from both prompts increases. The table uses separation minus control in percentage points and paired, pointwise 95% intervals for this panel."
|
| 117 |
+
],
|
| 118 |
+
"zh": [
|
| 119 |
+
"120 个参考程序家族在三至六个共享样例下各生成一对任务,并各加一个对照或分离样例。两种模型为 Qwen3-8B 和 Olmo-3-7B-Think,共有 1,440 条新轨迹和 480 条先前保存的五共享样例轨迹。三个、四个共享样例时各有 119 个家族通过配对认证,五个、六个时各有 120 个。",
|
| 120 |
+
"五个共享样例时,Qwen 在 120 道对照题中完成 67 道,在分离题中完成 12 道;Olmo 分别完成 52 和 15 道。两种模型通过另一对完全未展示的见证样例的概率均有上升。表中差值均为分离减对照,单位是百分点,区间为此面板按家族配对的逐项 95% 区间。"
|
| 121 |
+
]
|
| 122 |
+
},
|
| 123 |
+
"blocks": [
|
| 124 |
+
{
|
| 125 |
+
"type": "table",
|
| 126 |
+
"columns": [
|
| 127 |
+
{
|
| 128 |
+
"en": "Model",
|
| 129 |
+
"zh": "模型"
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"en": "Displayed, control → separation",
|
| 133 |
+
"zh": "题面,对照 → 分离"
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"en": "Displayed Δ (pp)",
|
| 137 |
+
"zh": "题面差值(百分点)"
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"en": "Hidden pair Δ (pp)",
|
| 141 |
+
"zh": "隐藏配对差值(百分点)"
|
| 142 |
+
}
|
| 143 |
+
],
|
| 144 |
+
"rows": [
|
| 145 |
+
[
|
| 146 |
+
"Qwen3-8B",
|
| 147 |
+
"67 → 12 / 120",
|
| 148 |
+
"−45.83 [−55.00, −36.67]",
|
| 149 |
+
"+9.17 [+4.17, +15.00]"
|
| 150 |
+
],
|
| 151 |
+
[
|
| 152 |
+
"Olmo-3-7B-Think",
|
| 153 |
+
"52 → 15 / 120",
|
| 154 |
+
"−30.83 [−40.83, −20.83]",
|
| 155 |
+
"+11.67 [+6.67, +17.50]"
|
| 156 |
+
]
|
| 157 |
+
],
|
| 158 |
+
"caption": {
|
| 159 |
+
"en": "Five shared examples plus the changed example. Across all eight model and base-count settings, the displayed effect ranges from −45.83 to −23.53 points and the hidden-pair effect from +8.40 to +13.45.",
|
| 160 |
+
"zh": "五个共享样例加上一个更换样例。全部八个模型与样例数设置下,题面差值为 −45.83 至 −23.53 个百分点,隐藏配对差值为 +8.40 至 +13.45 个百分点。"
|
| 161 |
+
},
|
| 162 |
+
"source": "https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/results/PRIMARY_PANEL_READABLE.csv"
|
| 163 |
+
}
|
| 164 |
+
]
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"id": "beyond-the-witness",
|
| 168 |
+
"title": {
|
| 169 |
+
"en": "What happens on other strings?",
|
| 170 |
+
"zh": "换到更广的字符串输入会怎样?"
|
| 171 |
+
},
|
| 172 |
+
"paragraphs": {
|
| 173 |
+
"en": [
|
| 174 |
+
"The hidden pair was chosen to distinguish programs. A second evaluation draws 32 uniform strings and 32 strings that activate both reference rules for each family. Uniform-input agreement falls in every model and base-count setting. Of the 3,840 uniform inputs, 3,325 map to themselves under the reference.",
|
| 175 |
+
"On the rule-active inputs the absolute agreement of separation programs is only 1.93–5.78%. At base five, Qwen reaches 2.76% and Olmo 2.01%. A gain on targeted witnesses therefore supports a narrower conclusion than recovering the reference transformation across inputs."
|
| 176 |
+
],
|
| 177 |
+
"zh": [
|
| 178 |
+
"隐藏见证配对是为区分程序而选择的。第二组评估则为每个家族抽取 32 ��均匀输入和 32 个能触发两条规则的输入。在全部模型和共享样例数设置下,均匀输入上的一致率都下降。3,840 个均匀输入中,有 3,325 个经过参考程序后保持原样。",
|
| 179 |
+
"在实际触发两条规则的新输入上,分离题程序的绝对一致率只有 1.93% 至 5.78%。五个共享样例时,Qwen 为 2.76%,Olmo 为 2.01%。定向见证上的进步说明模型行为发生了变化,广泛恢复参考变换仍很困难。"
|
| 180 |
+
]
|
| 181 |
+
},
|
| 182 |
+
"blocks": []
|
| 183 |
+
},
|
| 184 |
+
{
|
| 185 |
+
"id": "identity",
|
| 186 |
+
"title": {
|
| 187 |
+
"en": "An unchanged output can rule out a shortcut",
|
| 188 |
+
"zh": "输出没变的样例也能排除捷径"
|
| 189 |
+
},
|
| 190 |
+
"paragraphs": {
|
| 191 |
+
"en": [
|
| 192 |
+
"In a separate archived family, the reference first replaces f with fdd and then db with c. The pair iyxbdc → iyxbdc looks uneventful. A competing rule b → dc predicts iyxdcdc. Keeping this identity example excludes that candidate; removing it at the smaller base counts lets a one-rule solution return. This family was selected from its certificate transition before inspecting the model outcomes."
|
| 193 |
+
],
|
| 194 |
+
"zh": [
|
| 195 |
+
"另一组归档任务的参考规则是先把 f 换成 fdd,再把 db 换成 c。样例 iyxbdc → iyxbdc 看起来毫无变化,但候选规则 b → dc 会输出 iyxdcdc。保留这个恒等样例就能排除该候选,在较少共享样例的设置中去掉它,单规则解便重新出现。研究依据证书的变化选中这个家族,选择过程独立于模型表现。"
|
| 196 |
+
]
|
| 197 |
+
},
|
| 198 |
+
"blocks": []
|
| 199 |
+
},
|
| 200 |
+
{
|
| 201 |
+
"id": "replay",
|
| 202 |
+
"title": {
|
| 203 |
+
"en": "Replay the evidence",
|
| 204 |
+
"zh": "复核证据"
|
| 205 |
+
},
|
| 206 |
+
"paragraphs": {
|
| 207 |
+
"en": [
|
| 208 |
+
"Extract the release, enter its top-level folder and run the first two commands with Python 3.11 or later. The third uses NumPy and writes to a new directory. It verifies 960 certificates, 7,680 reference labels and the saved programs, then compares the three reconstructed tables with their archived counterparts.",
|
| 209 |
+
"The release supplies normalized trajectories, task specifications and executable checks. Earlier discovery and timed-search studies have separate coverage in the result map. The historical sampling chain of the 1,008-task retrospective roster remains unresolved. The companion reports the scope of those records separately from the matched 120-family experiment."
|
| 210 |
+
],
|
| 211 |
+
"zh": [
|
| 212 |
+
"解压版本包并进入顶层目录。用 Python 3.11 或更新版本运行前两条命令即可核对文件和图 1 实例。第三条使用 NumPy,将结果写入新目录;它复核 960 份证书、7,680 个参考标签与保存的程序,并把三张重建结果表逐一对照归档。",
|
| 213 |
+
"归档直接提供规范化轨迹、任务规格与可执行核验。早期探索与限时搜索等历史实验的材料覆盖范围列在结果映射中。另有 1,008 道回溯任务,其早期抽样链尚未确定;这部分记录与主要的 120 家族配对实验分别说明。"
|
| 214 |
+
]
|
| 215 |
+
},
|
| 216 |
+
"blocks": [
|
| 217 |
+
{
|
| 218 |
+
"type": "code",
|
| 219 |
+
"code": "python -B verify_manifest.py\npython -B replay_fixture.py\npython -B reproduce.py --out replay"
|
| 220 |
+
}
|
| 221 |
+
]
|
| 222 |
+
}
|
| 223 |
+
],
|
| 224 |
+
"sources": [
|
| 225 |
+
{
|
| 226 |
+
"label": {
|
| 227 |
+
"en": "Paper and version history · arXiv",
|
| 228 |
+
"zh": "预印本与版本记录 · arXiv"
|
| 229 |
+
},
|
| 230 |
+
"url": "https://arxiv.org/abs/2611.24975"
|
| 231 |
+
},
|
| 232 |
+
{
|
| 233 |
+
"label": {
|
| 234 |
+
"en": "Version 1 with appendix · HTML",
|
| 235 |
+
"zh": "v1 全文与附录 · HTML"
|
| 236 |
+
},
|
| 237 |
+
"url": "https://arxiv.org/html/2611.24975v1"
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"label": {
|
| 241 |
+
"en": "Figure 1 executable fixture",
|
| 242 |
+
"zh": "图 1 可执行实例"
|
| 243 |
+
},
|
| 244 |
+
"url": "https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/fixtures/end_to_end.json"
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"label": {
|
| 248 |
+
"en": "Primary and witness effects",
|
| 249 |
+
"zh": "主面板与隐藏配对结果"
|
| 250 |
+
},
|
| 251 |
+
"url": "https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/results/PRIMARY_PANEL_READABLE.csv"
|
| 252 |
+
},
|
| 253 |
+
{
|
| 254 |
+
"label": {
|
| 255 |
+
"en": "Uniform and active input panel",
|
| 256 |
+
"zh": "均匀与规则激活输入结果"
|
| 257 |
+
},
|
| 258 |
+
"url": "https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/results/BROADER_PANEL_READABLE.csv"
|
| 259 |
+
},
|
| 260 |
+
{
|
| 261 |
+
"label": {
|
| 262 |
+
"en": "Identity-example certificate",
|
| 263 |
+
"zh": "恒等样例证书"
|
| 264 |
+
},
|
| 265 |
+
"url": "https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/worked_case.json"
|
| 266 |
+
},
|
| 267 |
+
{
|
| 268 |
+
"label": {
|
| 269 |
+
"en": "Paper-to-artifact result map",
|
| 270 |
+
"zh": "论文至归档的结果映射"
|
| 271 |
+
},
|
| 272 |
+
"url": "https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/docs/result-map.csv"
|
| 273 |
+
},
|
| 274 |
+
{
|
| 275 |
+
"label": {
|
| 276 |
+
"en": "Interactive certificate explorer",
|
| 277 |
+
"zh": "交互式证书示例"
|
| 278 |
+
},
|
| 279 |
+
"url": "https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates"
|
| 280 |
+
}
|
| 281 |
+
]
|
| 282 |
+
}
|
docs/figures/construction.pdf
ADDED
|
Binary file (83.2 kB). View file
|
|
|
docs/figures/construction.png
ADDED
|
Git LFS Details
|
docs/maintenance.md
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Maintenance
|
| 2 |
+
|
| 3 |
+
The v1.0.0 tag identifies the frozen first artifact. The main branch can receive clarifications and verified examples. Changes to evidence or numerical outputs go into a separately versioned release with an updated file manifest and verification report.
|
| 4 |
+
|
| 5 |
+
The preprint, companion and license scope are linked in [the public-edition record](../PUBLICATION.md). Use the tagged tree for the scientific baseline and the public-edition bundle for the offline companions.
|
docs/release-manifest.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"title": "When Is Program Structure Required? Certificate-Grounded Calibration of Program-Induction Difficulty",
|
| 3 |
+
"author": "Wenzhang Du",
|
| 4 |
+
"affiliation": "Independent Researcher, China",
|
| 5 |
+
"version": "1.0.0",
|
| 6 |
+
"date": "2026-11-16",
|
| 7 |
+
"repository": "https://github.com/dqswordman/program-structure-certificates",
|
| 8 |
+
"frozen_tag": "v1.0.0",
|
| 9 |
+
"arxiv_id": "2611.24975",
|
| 10 |
+
"arxiv_url": "https://arxiv.org/abs/2611.24975",
|
| 11 |
+
"arxiv_version": "v1",
|
| 12 |
+
"frozen_commit": "8b0f2e0dd17ee36ce9233a9053dc55c22110dfce",
|
| 13 |
+
"companion_published": "2026-11-20",
|
| 14 |
+
"companion_url": "https://dqswordman.github.io/technical-reports/program-structure-certificates/",
|
| 15 |
+
"space_url": "https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates",
|
| 16 |
+
"manuscript_sha256": "efc8e9c94d41ebfd31565321ae96f94ea86c91e925321d2a16c599ef8030fad0",
|
| 17 |
+
"arxiv_source_sha256": "e5151889675b146cc79f19118e8dfa66ec8b915c0063817398eb762dddb9062f",
|
| 18 |
+
"pages": 26,
|
| 19 |
+
"figures": 1,
|
| 20 |
+
"tables": 24,
|
| 21 |
+
"reproduction_report": "docs/reproduction_report.json",
|
| 22 |
+
"models": {
|
| 23 |
+
"M1": {
|
| 24 |
+
"repo_id": "Qwen/Qwen3-8B",
|
| 25 |
+
"revision": "b968826d9c46dd6066d109eabc6255188de91218"
|
| 26 |
+
},
|
| 27 |
+
"M2": {
|
| 28 |
+
"repo_id": "allenai/Olmo-3-7B-Think",
|
| 29 |
+
"revision": "d97e442d7cc678210054dbcc9b440894d62c89a4"
|
| 30 |
+
}
|
| 31 |
+
}
|
| 32 |
+
}
|
docs/report.en.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# When does an example force a second rule?
|
| 2 |
+
|
| 3 |
+
Wenzhang Du · arXiv:2611.24975v1 · v1.0 · 2026-11-20
|
| 4 |
+
|
| 5 |
+
A model can explain every demonstration with a shortcut. One carefully chosen example lets us certify when that shortcut disappears, then measure what the model does on the task and on new strings.
|
| 6 |
+
|
| 7 |
+
## Change one example, keep the reference program
|
| 8 |
+
|
| 9 |
+
Imagine inferring a program from six input–output pairs. Five pairs are shared. In the control, the last pair maps iiiexa to iiiazy. The single replacement exa → azy fits all six. The same observations also admit a two-step explanation: replace ex with azw, then wa with y.
|
| 10 |
+
|
| 11 |
+
In the matched separation task the final pair becomes wbawex → wbayzw. The shortcut leaves wbawex unchanged. The two-step program produces wbawazw after its first rule and wbayzw after its second. All other demonstrations stay fixed. The figure traces both explanations on this archived task.
|
| 12 |
+
|
| 13 |
+

|
| 14 |
+
|
| 15 |
+
Figure 1. Both tasks use the same two-rule reference. The final demonstration determines whether a one-rule solution remains possible.
|
| 16 |
+
|
| 17 |
+
## What the certificate proves
|
| 18 |
+
|
| 19 |
+
Here a rule replaces every non-overlapping occurrence of a literal string. Rules run in sequence. The study fixes a finite alphabet, a left-hand side of one to three characters and a right-hand side of up to three. A changed output rules out the empty program. The verifier enumerates the legal one-rule explanations and executes a two-rule witness.
|
| 20 |
+
|
| 21 |
+
For this example one single-rule program survives in the control and none survives in separation. The successful witness gives an upper bound of two, so the minimum lengths are exactly one and two. Certification uses the displayed pairs before inference. This statement depends on the chosen language and rule lengths.
|
| 22 |
+
|
| 23 |
+
## Solving the prompt and passing the hidden pair
|
| 24 |
+
|
| 25 |
+
Each of the 120 reference families supplies matched control and separation tasks with three to six shared examples, plus one arm-specific example. The panel contains 1,440 new trajectories and 480 previously saved base-five trajectories for Qwen3-8B and Olmo-3-7B-Think. The certified comparison uses 119 families at bases three and four and all 120 at bases five and six.
|
| 26 |
+
|
| 27 |
+
With five shared examples, Qwen completes 67 of 120 control tasks and 12 separation tasks. Olmo completes 52 and 15. In both models, success on a separate witness pair hidden from both prompts increases. The table uses separation minus control in percentage points and paired, pointwise 95% intervals for this panel.
|
| 28 |
+
|
| 29 |
+
| Model | Displayed, control → separation | Displayed Δ (pp) | Hidden pair Δ (pp) |
|
| 30 |
+
| --- | --- | --- | --- |
|
| 31 |
+
| Qwen3-8B | 67 → 12 / 120 | −45.83 [−55.00, −36.67] | +9.17 [+4.17, +15.00] |
|
| 32 |
+
| Olmo-3-7B-Think | 52 → 15 / 120 | −30.83 [−40.83, −20.83] | +11.67 [+6.67, +17.50] |
|
| 33 |
+
|
| 34 |
+
Five shared examples plus the changed example. Across all eight model and base-count settings, the displayed effect ranges from −45.83 to −23.53 points and the hidden-pair effect from +8.40 to +13.45.
|
| 35 |
+
|
| 36 |
+
## What happens on other strings?
|
| 37 |
+
|
| 38 |
+
The hidden pair was chosen to distinguish programs. A second evaluation draws 32 uniform strings and 32 strings that activate both reference rules for each family. Uniform-input agreement falls in every model and base-count setting. Of the 3,840 uniform inputs, 3,325 map to themselves under the reference.
|
| 39 |
+
|
| 40 |
+
On the rule-active inputs the absolute agreement of separation programs is only 1.93–5.78%. At base five, Qwen reaches 2.76% and Olmo 2.01%. A gain on targeted witnesses therefore supports a narrower conclusion than recovering the reference transformation across inputs.
|
| 41 |
+
|
| 42 |
+
## An unchanged output can rule out a shortcut
|
| 43 |
+
|
| 44 |
+
In a separate archived family, the reference first replaces f with fdd and then db with c. The pair iyxbdc → iyxbdc looks uneventful. A competing rule b → dc predicts iyxdcdc. Keeping this identity example excludes that candidate; removing it at the smaller base counts lets a one-rule solution return. This family was selected from its certificate transition before inspecting the model outcomes.
|
| 45 |
+
|
| 46 |
+
## Replay the evidence
|
| 47 |
+
|
| 48 |
+
Extract the release, enter its top-level folder and run the first two commands with Python 3.11 or later. The third uses NumPy and writes to a new directory. It verifies 960 certificates, 7,680 reference labels and the saved programs, then compares the three reconstructed tables with their archived counterparts.
|
| 49 |
+
|
| 50 |
+
The release supplies normalized trajectories, task specifications and executable checks. Earlier discovery and timed-search studies have separate coverage in the result map. The historical sampling chain of the 1,008-task retrospective roster remains unresolved. The companion reports the scope of those records separately from the matched 120-family experiment.
|
| 51 |
+
|
| 52 |
+
```sh
|
| 53 |
+
python -B verify_manifest.py
|
| 54 |
+
python -B replay_fixture.py
|
| 55 |
+
python -B reproduce.py --out replay
|
| 56 |
+
```
|
| 57 |
+
|
| 58 |
+
## Sources
|
| 59 |
+
|
| 60 |
+
- [Paper and version history · arXiv](https://arxiv.org/abs/2611.24975)
|
| 61 |
+
- [Version 1 with appendix · HTML](https://arxiv.org/html/2611.24975v1)
|
| 62 |
+
- [Figure 1 executable fixture](https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/fixtures/end_to_end.json)
|
| 63 |
+
- [Primary and witness effects](https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/results/PRIMARY_PANEL_READABLE.csv)
|
| 64 |
+
- [Uniform and active input panel](https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/results/BROADER_PANEL_READABLE.csv)
|
| 65 |
+
- [Identity-example certificate](https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/worked_case.json)
|
| 66 |
+
- [Paper-to-artifact result map](https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/docs/result-map.csv)
|
| 67 |
+
- [Interactive certificate explorer](https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates)
|
docs/report.zh.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 什么样的样例会迫使程序使用第二条规则?
|
| 2 |
+
|
| 3 |
+
Wenzhang Du · arXiv:2611.24975v1 · v1.0 · 2026-11-20
|
| 4 |
+
|
| 5 |
+
模型可能靠一条捷径解释题面上的所有样例。这项研究只替换一个样例,严格核验第二条规则何时成为必要条件,再分别检查模型对题面和新字符串的判断。
|
| 6 |
+
|
| 7 |
+
## 同一个参考程序,只换一道样例
|
| 8 |
+
|
| 9 |
+
试着从六组输入输出中推断程序。两道题共享前五组。对照题的最后一组是 iiiexa → iiiazy,用一条 exa → azy 就能解释六组样例。另一个解释有两步:先把 ex 换成 azw,再把 wa 换成 y。
|
| 10 |
+
|
| 11 |
+
在配对的分离题中,把最后一组换成 wbawex → wbayzw。单规则捷径会原样保留输入;两步程序先得到 wbawazw,再得到 wbayzw。其余五组样例全部相同。下图给出了这对归档任务的完整构造。
|
| 12 |
+
|
| 13 |
+

|
| 14 |
+
|
| 15 |
+
图 1。两道题共用同一个双规则参考程序;最后一组样例决定单规则解是否仍然存在。
|
| 16 |
+
|
| 17 |
+
## 证书究竟证明了什么
|
| 18 |
+
|
| 19 |
+
每条规则替换字符串中所有互不重叠的字面匹配,多条规则按顺序执行。研究固定有限字母表,左侧长度为一至三个字符,右侧长度最多三个字符。一组发生变化的输出即可排除空程序。验证器随后枚举全部合法单规则解释,并实际执行一个双规则解。
|
| 20 |
+
|
| 21 |
+
在图中,对照题恰好保留一个单规则解,分离题的单规则解数为零。双规则程序在两题均通过,因此最短长度分别精确为一与二。认证在模型推理之前完成,结论的范围由这套替换语言及规则长度界定。
|
| 22 |
+
|
| 23 |
+
## 题面完成率和隐藏配对的成功率
|
| 24 |
+
|
| 25 |
+
120 个参考程序家族在三至六个共享样例下各生成一对任务,并各加一个对照或分离样例。两种模型为 Qwen3-8B 和 Olmo-3-7B-Think,共有 1,440 条新轨迹和 480 条先前保存的五共享样例轨迹。三个、四个共享样例时各有 119 个家族通过配对认证,五个、六个时各有 120 个。
|
| 26 |
+
|
| 27 |
+
五个共享样例时,Qwen 在 120 道对照题中完成 67 道,在分离题中完成 12 道;Olmo 分别完成 52 和 15 道。两种模型通过另一对完全未展示的见证样例的概率均有上升。表中差值均为分离减对照,单位是百分点,区间为此面板按家族配对的逐项 95% 区间。
|
| 28 |
+
|
| 29 |
+
| 模型 | 题面,对照 → 分离 | 题面差值(百分点) | 隐藏配对差值(百分点) |
|
| 30 |
+
| --- | --- | --- | --- |
|
| 31 |
+
| Qwen3-8B | 67 → 12 / 120 | −45.83 [−55.00, −36.67] | +9.17 [+4.17, +15.00] |
|
| 32 |
+
| Olmo-3-7B-Think | 52 → 15 / 120 | −30.83 [−40.83, −20.83] | +11.67 [+6.67, +17.50] |
|
| 33 |
+
|
| 34 |
+
五个共享样例加上一个更换样例。全部八个模型与样例数设置下,题面差值为 −45.83 至 −23.53 个百分点,隐藏配对差值为 +8.40 至 +13.45 个百分点。
|
| 35 |
+
|
| 36 |
+
## 换到更广的字符串输入会怎样?
|
| 37 |
+
|
| 38 |
+
隐藏见证配对是为区分程序而选择的。第二组评估则为每个家族抽取 32 个均匀输入和 32 个能触发两条规则的输入。在全部模型和共享样例数设置下,均匀输入上的一致率都下降。3,840 个均匀输入中,有 3,325 个经过参考程序后保持原样。
|
| 39 |
+
|
| 40 |
+
在实际触发两条规则的新输入上,分离题程序的绝对一致率只有 1.93% 至 5.78%。五个共享样例时,Qwen 为 2.76%,Olmo 为 2.01%。定向见证上的进步说明模型行为发生了变化,广泛恢复参考变换仍很困难。
|
| 41 |
+
|
| 42 |
+
## 输出没变的样例也能排除捷径
|
| 43 |
+
|
| 44 |
+
另一组归档任务的参考规则是先把 f 换成 fdd,再把 db 换成 c。样例 iyxbdc → iyxbdc 看起来毫无变化,但候选规则 b → dc 会输出 iyxdcdc。保留这个恒等样例就能排除该候选,在较少共享样例的设置中去掉它,单规则解便重新出现。研究依据证书的变化选中这个家族,选择过程独立于模型表现。
|
| 45 |
+
|
| 46 |
+
## 复核证据
|
| 47 |
+
|
| 48 |
+
解压版本包并进入顶层目录。用 Python 3.11 或更新版本运行前两条命令即可核对文件和图 1 实例。第三条使用 NumPy,将结果写入新目录;它复核 960 份证书、7,680 个参考标签与保存的程序,并把三张重建结果表逐一对照归档。
|
| 49 |
+
|
| 50 |
+
归档直接提供规范化轨迹、任务规格与可执行核验。早期探索与限时搜索等历史实验的材料覆盖范围列在结果映射中。另有 1,008 道回溯任务,其早期抽样链尚未确定;这部分记录与主要的 120 家族配对实验分别说明。
|
| 51 |
+
|
| 52 |
+
```sh
|
| 53 |
+
python -B verify_manifest.py
|
| 54 |
+
python -B replay_fixture.py
|
| 55 |
+
python -B reproduce.py --out replay
|
| 56 |
+
```
|
| 57 |
+
|
| 58 |
+
## 资料来源
|
| 59 |
+
|
| 60 |
+
- [预印本与版本记录 · arXiv](https://arxiv.org/abs/2611.24975)
|
| 61 |
+
- [v1 全文与附录 · HTML](https://arxiv.org/html/2611.24975v1)
|
| 62 |
+
- [图 1 可执行实例](https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/fixtures/end_to_end.json)
|
| 63 |
+
- [主面板与隐藏配对结果](https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/results/PRIMARY_PANEL_READABLE.csv)
|
| 64 |
+
- [均匀与规则激活输入结果](https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/results/BROADER_PANEL_READABLE.csv)
|
| 65 |
+
- [恒等样例证书](https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/worked_case.json)
|
| 66 |
+
- [论文至归档的结果映射](https://github.com/dqswordman/program-structure-certificates/blob/v1.0.0/docs/result-map.csv)
|
| 67 |
+
- [交互式证书示例](https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates)
|
docs/reproduction.md
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reproduction
|
| 2 |
+
|
| 3 |
+
Use [v1.0.0](https://github.com/dqswordman/program-structure-certificates/tree/v1.0.0) for the frozen research artifact. Run the commands from the repository root.
|
| 4 |
+
|
| 5 |
+
## Quick check
|
| 6 |
+
|
| 7 |
+
Python 3.11 or newer suffices for both commands.
|
| 8 |
+
|
| 9 |
+
python -B verify_manifest.py
|
| 10 |
+
python -B replay_fixture.py
|
| 11 |
+
|
| 12 |
+
Both print PASS. The fixture prints the certificate minima for the Figure 1 task pair, and the displayed, witness, uniform, and rule-active successes of four archived programs. The fixture starts from the exact task strings, checks each selected program against the stored parser output, and rebuilds the evaluation counts.
|
| 13 |
+
|
| 14 |
+
## Full replay
|
| 15 |
+
|
| 16 |
+
The validated environment used Python 3.12.7 and NumPy 1.26.4. Install the NumPy dependency in your environment before replay.
|
| 17 |
+
|
| 18 |
+
python -m pip install -r requirements.txt
|
| 19 |
+
python -B reproduce.py --out replay
|
| 20 |
+
|
| 21 |
+
Choose a new output directory each time. The replay enumerates **960** certificates for **480** pairs, checks all **7,680** wider reference labels and both rules' activation on the **3,840** active inputs, then recertifies the **1,008** archived tasks. It reconstructs **1,920** per-trajectory rows, **348** paired-effect rows, and **2,872** family-difference rows. Each CSV is compared with its archived counterpart under results/. Expected status and file digests appear in [the verification report](reproduction_report.json).
|
| 22 |
+
|
| 23 |
+
The full replay uses text and selected programs from precomputed trajectories. Model checkpoints and neural inference are outside this replay. The generation protocol and exact model revisions are in [the model contract](../data/model_contract.json).
|
| 24 |
+
|
| 25 |
+
## Paper coverage
|
| 26 |
+
|
| 27 |
+
The base-count panel uses shared family IDs and the original paired-family analysis. At bases three and four the certified cohort contains 119 families, while at bases five and six it contains 120. The base-five point estimates for Qwen and Olmo correspond to the original held-out evaluation. The original adjusted intervals and the historical studies listed in [release scope](../RELEASE_SCOPE.txt) are documented separately. [Result map](result-map.csv) · [Readout and bootstrap contract](../data/analysis_contract.json)
|
docs/reproduction_report.json
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "PASS",
|
| 3 |
+
"scientific_tables": [
|
| 4 |
+
{
|
| 5 |
+
"table": "per_family_metrics.csv",
|
| 6 |
+
"rows": 1920,
|
| 7 |
+
"numeric_and_categorical_values_match": true,
|
| 8 |
+
"CSV_bytes_match": true,
|
| 9 |
+
"sha256": "18e2ebd712fd2cebe6f7e766acc2d67707d29eebbf5a28ae00a0ed34aee008d6"
|
| 10 |
+
},
|
| 11 |
+
{
|
| 12 |
+
"table": "paired_effects.csv",
|
| 13 |
+
"rows": 348,
|
| 14 |
+
"numeric_and_categorical_values_match": true,
|
| 15 |
+
"CSV_bytes_match": true,
|
| 16 |
+
"sha256": "5085090c3147e28f00184546201d6b1d7e06b7bcc70b42a9e468625b266f4b90"
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"table": "family_paired_differences.csv",
|
| 20 |
+
"rows": 2872,
|
| 21 |
+
"numeric_and_categorical_values_match": true,
|
| 22 |
+
"CSV_bytes_match": true,
|
| 23 |
+
"sha256": "10aacaa32c80bef20442cb7d1aadd97fe1cccbb2131dba0d85ca525bb787b9f3"
|
| 24 |
+
}
|
| 25 |
+
],
|
| 26 |
+
"checks": {
|
| 27 |
+
"panel_certificates_checked": 960,
|
| 28 |
+
"panel_variants_checked": 480,
|
| 29 |
+
"broader_reference_labels_checked": 7680,
|
| 30 |
+
"active_inputs_with_both_rules_verified": 3840,
|
| 31 |
+
"native_archive_rows_checked": 1008,
|
| 32 |
+
"certificate_counts": {
|
| 33 |
+
"(3, 'SHORTCUT_RETURN_1_TO_1')": 1,
|
| 34 |
+
"(4, 'SHORTCUT_RETURN_1_TO_1')": 1,
|
| 35 |
+
"(5, 'CERT_1_TO_2')": 120,
|
| 36 |
+
"(6, 'CERT_1_TO_2')": 120,
|
| 37 |
+
"(3, 'CERT_1_TO_2')": 119,
|
| 38 |
+
"(4, 'CERT_1_TO_2')": 119
|
| 39 |
+
},
|
| 40 |
+
"native_archive_counts": {
|
| 41 |
+
"('NO_ONE_RULE_BOUND2_TO_K', 3)": 250,
|
| 42 |
+
"('NO_ONE_RULE_BOUND2_TO_K', 4)": 253,
|
| 43 |
+
"('CERTIFIED_MINIMUM2', 2)": 226,
|
| 44 |
+
"('NO_ONE_RULE_BOUND2_TO_K', 5)": 256,
|
| 45 |
+
"('ONE_RULE_SHORTCUT', 2)": 20,
|
| 46 |
+
"('ONE_RULE_SHORTCUT', 3)": 3
|
| 47 |
+
},
|
| 48 |
+
"worked_case": {
|
| 49 |
+
"family_id": "5afeb1c4a228abe8751d0aff13ea1896ea3907e961e175fc43f83b5e0168e88c",
|
| 50 |
+
"verified": true,
|
| 51 |
+
"reference_identity_output": "iyxbdc",
|
| 52 |
+
"shortcut_output": "iyxdcdc",
|
| 53 |
+
"separation_minimum_by_base": {
|
| 54 |
+
"3": 1,
|
| 55 |
+
"4": 1,
|
| 56 |
+
"5": 2
|
| 57 |
+
},
|
| 58 |
+
"selection": "Unique certificate-defined shortcut-return family; no model outcomes used."
|
| 59 |
+
}
|
| 60 |
+
},
|
| 61 |
+
"scored_requests": 1920,
|
| 62 |
+
"paired_effect_rows": 348,
|
| 63 |
+
"common_certified_family_count": 119,
|
| 64 |
+
"new_observation_counts": {
|
| 65 |
+
"rows": 1440,
|
| 66 |
+
"EOS": 544,
|
| 67 |
+
"tokens": 10268836,
|
| 68 |
+
"unparseable": 899,
|
| 69 |
+
"CAP": 896
|
| 70 |
+
},
|
| 71 |
+
"numpy_version": "1.26.4",
|
| 72 |
+
"CPU_elapsed_seconds": 5.75851109996438,
|
| 73 |
+
"model_or_GPU_calls": 0,
|
| 74 |
+
"original_raw_JSON_byte_layout_reconstructed": false,
|
| 75 |
+
"normalization_scope": "Scientific contents and score tables are reproduced from the normalized observations."
|
| 76 |
+
}
|
docs/result-map.csv
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
paper_location,page,result,input,command,output,verification_coverage
|
| 2 |
+
Figure 1,2,Worked one-to-two task pair and four saved programs,"data/families.jsonl, data/base_variants.jsonl, fixtures/end_to_end.json",python -B replay_fixture.py,fixture JSON on standard output,CPU-verified
|
| 3 |
+
Table 1,6,Original validation displayed completion and common-nine agreement,"data/trajectories.jsonl.gz, data/analysis_contract.json",python -B reproduce.py --out replay,"replay/paired_effects.csv, b=5",Qwen/Olmo point estimates. Earlier adjusted intervals and Phi appear as manuscript summaries.
|
| 4 |
+
Table 2,6,Original isolated witness pair,"data/trajectories.jsonl.gz, data/analysis_contract.json",python -B reproduce.py --out replay,"replay/paired_effects.csv, unshown_pair_success at b=5",Qwen/Olmo point estimates. Earlier adjusted intervals and Phi appear as manuscript summaries.
|
| 5 |
+
Table 3,7,Certified contrasts across base-example counts,"data/certificate_records.jsonl, data/trajectories.jsonl.gz",python -B reproduce.py --out replay,"replay/paired_effects.csv, results/PRIMARY_PANEL_READABLE.csv",CPU-verified
|
| 6 |
+
Table 4,7,Uniform and rule-active paired changes,"data/broader_inputs.jsonl, data/trajectories.jsonl.gz",python -B reproduce.py --out replay,"replay/paired_effects.csv, results/BROADER_PANEL_READABLE.csv",CPU-verified
|
| 7 |
+
Table 5,8,Bounded timed symbolic search,Historical solver protocol,,manuscript table,Documented historical summary
|
| 8 |
+
Table 17,23,"Retrospective audit of the 1,008 task archive",data/native1008.csv,python -B reproduce.py --out replay,"replay/REPLAY_VALIDATION.json, native_archive_counts",CPU-verified
|
| 9 |
+
Table 19,24,Qwen endpoint counts by example count,"data/base_variants.jsonl, data/trajectories.jsonl.gz",python -B reproduce.py --out replay,"replay/per_family_metrics.csv, replay/paired_effects.csv",CPU-verified
|
| 10 |
+
Table 20,25,Olmo endpoint counts by example count,"data/base_variants.jsonl, data/trajectories.jsonl.gz",python -B reproduce.py --out replay,"replay/per_family_metrics.csv, replay/paired_effects.csv",CPU-verified
|
| 11 |
+
Appendix K,21,Reproduction scope and repository contents,"MANIFEST.json, RELEASE_SCOPE.txt",python -B verify_manifest.py,manifest verification on standard output,Current file inventory
|
explore.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
async function activate() {
|
| 2 |
+
const response = await fetch('explorer.json');
|
| 3 |
+
if (!response.ok) return;
|
| 4 |
+
const { fixture, panel } = await response.json();
|
| 5 |
+
const byId = id => document.getElementById(id);
|
| 6 |
+
function showCase() {
|
| 7 |
+
const arm = document.querySelector('[name="arm"]:checked').value;
|
| 8 |
+
const task = fixture.task_pair[arm];
|
| 9 |
+
const input = task.inputs.at(-1);
|
| 10 |
+
const cert = fixture.expected.certificates[arm];
|
| 11 |
+
byId('input').textContent = input;
|
| 12 |
+
byId('target').textContent = task.outputs.at(-1);
|
| 13 |
+
byId('shortcut').textContent = input.replaceAll('exa','azy');
|
| 14 |
+
const n = cert.complete_one_rule_solutions.length;
|
| 15 |
+
byId('verdict').textContent = (arm === 'CONTROL' ? 'Control' : 'Separation') + ' · ' + n + ' single-rule solution' + (n === 1 ? ' fits' : 's fit') + ' all six demonstrations. Certified minimum length ' + cert.minimum_length + '.';
|
| 16 |
+
}
|
| 17 |
+
function showPanel() {
|
| 18 |
+
const row = panel.find(r => r.model === byId('model').value && r.base_count === byId('base').value);
|
| 19 |
+
byId('counts').textContent = row.primary_family_N + ' families · ' + row.CONTROL_displayed_successes + ' control and ' + row.SEPARATION_displayed_successes + ' separation tasks completed.';
|
| 20 |
+
byId('displayed').textContent = row.displayed_delta_pp_and_95CI.replaceAll('-', '−');
|
| 21 |
+
byId('hidden-pair').textContent = row.unshown_pair_delta_pp_and_95CI.replaceAll('-', '−');
|
| 22 |
+
}
|
| 23 |
+
byId('arms').addEventListener('change',showCase);
|
| 24 |
+
byId('models').addEventListener('change',showPanel);
|
| 25 |
+
showCase(); showPanel();
|
| 26 |
+
byId('arms').hidden = byId('models').hidden = false;
|
| 27 |
+
}
|
| 28 |
+
activate().catch(() => {});
|
explorer.json
ADDED
|
@@ -0,0 +1,338 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fixture": {
|
| 3 |
+
"purpose": "End-to-end replay of the existing Figure 1 family, selected by its role in the paper; uses both models and arms at base five, without choosing outputs by success.",
|
| 4 |
+
"sample_id": "3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e",
|
| 5 |
+
"base_count": 5,
|
| 6 |
+
"reference_rule_pairs": [
|
| 7 |
+
[
|
| 8 |
+
"ex",
|
| 9 |
+
"azw"
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"wa",
|
| 13 |
+
"y"
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"task_pair": {
|
| 17 |
+
"CONTROL": {
|
| 18 |
+
"inputs": [
|
| 19 |
+
"gud",
|
| 20 |
+
"czua",
|
| 21 |
+
"bwcied",
|
| 22 |
+
"dzexa",
|
| 23 |
+
"yf",
|
| 24 |
+
"iiiexa"
|
| 25 |
+
],
|
| 26 |
+
"outputs": [
|
| 27 |
+
"gud",
|
| 28 |
+
"czua",
|
| 29 |
+
"bwcied",
|
| 30 |
+
"dzazy",
|
| 31 |
+
"yf",
|
| 32 |
+
"iiiazy"
|
| 33 |
+
]
|
| 34 |
+
},
|
| 35 |
+
"SEPARATION": {
|
| 36 |
+
"inputs": [
|
| 37 |
+
"gud",
|
| 38 |
+
"czua",
|
| 39 |
+
"bwcied",
|
| 40 |
+
"dzexa",
|
| 41 |
+
"yf",
|
| 42 |
+
"wbawex"
|
| 43 |
+
],
|
| 44 |
+
"outputs": [
|
| 45 |
+
"gud",
|
| 46 |
+
"czua",
|
| 47 |
+
"bwcied",
|
| 48 |
+
"dzazy",
|
| 49 |
+
"yf",
|
| 50 |
+
"wbayzw"
|
| 51 |
+
]
|
| 52 |
+
}
|
| 53 |
+
},
|
| 54 |
+
"request_ids": [
|
| 55 |
+
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL",
|
| 56 |
+
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION",
|
| 57 |
+
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL",
|
| 58 |
+
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION"
|
| 59 |
+
],
|
| 60 |
+
"expected": {
|
| 61 |
+
"status": "PASS",
|
| 62 |
+
"family": "3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e",
|
| 63 |
+
"base_count": 5,
|
| 64 |
+
"certificates": {
|
| 65 |
+
"CONTROL": {
|
| 66 |
+
"complete_one_rule_solutions": [
|
| 67 |
+
[
|
| 68 |
+
"exa",
|
| 69 |
+
"azy"
|
| 70 |
+
]
|
| 71 |
+
],
|
| 72 |
+
"minimum_length": 1,
|
| 73 |
+
"two_rule_reference_passes": true
|
| 74 |
+
},
|
| 75 |
+
"SEPARATION": {
|
| 76 |
+
"complete_one_rule_solutions": [],
|
| 77 |
+
"minimum_length": 2,
|
| 78 |
+
"two_rule_reference_passes": true
|
| 79 |
+
}
|
| 80 |
+
},
|
| 81 |
+
"saved_outputs": {
|
| 82 |
+
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION": {
|
| 83 |
+
"selected_rules": null,
|
| 84 |
+
"legal_selected_program": false,
|
| 85 |
+
"targets": {
|
| 86 |
+
"displayed": {
|
| 87 |
+
"hits": 0,
|
| 88 |
+
"inputs": 6,
|
| 89 |
+
"all_pass": false
|
| 90 |
+
},
|
| 91 |
+
"common_nine": {
|
| 92 |
+
"hits": 0,
|
| 93 |
+
"inputs": 9,
|
| 94 |
+
"all_pass": false
|
| 95 |
+
},
|
| 96 |
+
"wholly_unshown_pair": {
|
| 97 |
+
"hits": 0,
|
| 98 |
+
"inputs": 2,
|
| 99 |
+
"all_pass": false
|
| 100 |
+
},
|
| 101 |
+
"uniform": {
|
| 102 |
+
"hits": 0,
|
| 103 |
+
"inputs": 32,
|
| 104 |
+
"all_pass": false
|
| 105 |
+
},
|
| 106 |
+
"active": {
|
| 107 |
+
"hits": 0,
|
| 108 |
+
"inputs": 32,
|
| 109 |
+
"all_pass": false
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
},
|
| 113 |
+
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL": {
|
| 114 |
+
"selected_rules": [
|
| 115 |
+
[
|
| 116 |
+
"a",
|
| 117 |
+
"y"
|
| 118 |
+
],
|
| 119 |
+
[
|
| 120 |
+
"e",
|
| 121 |
+
"a"
|
| 122 |
+
],
|
| 123 |
+
[
|
| 124 |
+
"x",
|
| 125 |
+
"z"
|
| 126 |
+
]
|
| 127 |
+
],
|
| 128 |
+
"legal_selected_program": true,
|
| 129 |
+
"targets": {
|
| 130 |
+
"displayed": {
|
| 131 |
+
"hits": 4,
|
| 132 |
+
"inputs": 6,
|
| 133 |
+
"all_pass": false
|
| 134 |
+
},
|
| 135 |
+
"common_nine": {
|
| 136 |
+
"hits": 5,
|
| 137 |
+
"inputs": 9,
|
| 138 |
+
"all_pass": false
|
| 139 |
+
},
|
| 140 |
+
"wholly_unshown_pair": {
|
| 141 |
+
"hits": 1,
|
| 142 |
+
"inputs": 2,
|
| 143 |
+
"all_pass": false
|
| 144 |
+
},
|
| 145 |
+
"uniform": {
|
| 146 |
+
"hits": 11,
|
| 147 |
+
"inputs": 32,
|
| 148 |
+
"all_pass": false
|
| 149 |
+
},
|
| 150 |
+
"active": {
|
| 151 |
+
"hits": 5,
|
| 152 |
+
"inputs": 32,
|
| 153 |
+
"all_pass": false
|
| 154 |
+
}
|
| 155 |
+
}
|
| 156 |
+
},
|
| 157 |
+
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION": {
|
| 158 |
+
"selected_rules": [
|
| 159 |
+
[
|
| 160 |
+
"exa",
|
| 161 |
+
"azy"
|
| 162 |
+
],
|
| 163 |
+
[
|
| 164 |
+
"wex",
|
| 165 |
+
"yzw"
|
| 166 |
+
]
|
| 167 |
+
],
|
| 168 |
+
"legal_selected_program": true,
|
| 169 |
+
"targets": {
|
| 170 |
+
"displayed": {
|
| 171 |
+
"hits": 6,
|
| 172 |
+
"inputs": 6,
|
| 173 |
+
"all_pass": true
|
| 174 |
+
},
|
| 175 |
+
"common_nine": {
|
| 176 |
+
"hits": 9,
|
| 177 |
+
"inputs": 9,
|
| 178 |
+
"all_pass": true
|
| 179 |
+
},
|
| 180 |
+
"wholly_unshown_pair": {
|
| 181 |
+
"hits": 2,
|
| 182 |
+
"inputs": 2,
|
| 183 |
+
"all_pass": true
|
| 184 |
+
},
|
| 185 |
+
"uniform": {
|
| 186 |
+
"hits": 32,
|
| 187 |
+
"inputs": 32,
|
| 188 |
+
"all_pass": true
|
| 189 |
+
},
|
| 190 |
+
"active": {
|
| 191 |
+
"hits": 0,
|
| 192 |
+
"inputs": 32,
|
| 193 |
+
"all_pass": false
|
| 194 |
+
}
|
| 195 |
+
}
|
| 196 |
+
},
|
| 197 |
+
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL": {
|
| 198 |
+
"selected_rules": null,
|
| 199 |
+
"legal_selected_program": false,
|
| 200 |
+
"targets": {
|
| 201 |
+
"displayed": {
|
| 202 |
+
"hits": 0,
|
| 203 |
+
"inputs": 6,
|
| 204 |
+
"all_pass": false
|
| 205 |
+
},
|
| 206 |
+
"common_nine": {
|
| 207 |
+
"hits": 0,
|
| 208 |
+
"inputs": 9,
|
| 209 |
+
"all_pass": false
|
| 210 |
+
},
|
| 211 |
+
"wholly_unshown_pair": {
|
| 212 |
+
"hits": 0,
|
| 213 |
+
"inputs": 2,
|
| 214 |
+
"all_pass": false
|
| 215 |
+
},
|
| 216 |
+
"uniform": {
|
| 217 |
+
"hits": 0,
|
| 218 |
+
"inputs": 32,
|
| 219 |
+
"all_pass": false
|
| 220 |
+
},
|
| 221 |
+
"active": {
|
| 222 |
+
"hits": 0,
|
| 223 |
+
"inputs": 32,
|
| 224 |
+
"all_pass": false
|
| 225 |
+
}
|
| 226 |
+
}
|
| 227 |
+
}
|
| 228 |
+
},
|
| 229 |
+
"model_inference_calls": 0
|
| 230 |
+
}
|
| 231 |
+
},
|
| 232 |
+
"panel": [
|
| 233 |
+
{
|
| 234 |
+
"model": "Qwen3-8B",
|
| 235 |
+
"base_count": "3",
|
| 236 |
+
"displayed_examples": "4",
|
| 237 |
+
"primary_family_N": "119",
|
| 238 |
+
"CONTROL_displayed_successes": "59",
|
| 239 |
+
"SEPARATION_displayed_successes": "15",
|
| 240 |
+
"displayed_delta_pp_and_95CI": "-36.97 [-46.22,-27.73]",
|
| 241 |
+
"unshown_pair_delta_pp_and_95CI": "+13.45 [+7.56,+20.17]",
|
| 242 |
+
"all_legal_N": "120",
|
| 243 |
+
"all_legal_displayed_delta_pp_and_95CI": "-36.67 [-45.83,-27.50]",
|
| 244 |
+
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"model": "Qwen3-8B",
|
| 248 |
+
"base_count": "4",
|
| 249 |
+
"displayed_examples": "5",
|
| 250 |
+
"primary_family_N": "119",
|
| 251 |
+
"CONTROL_displayed_successes": "61",
|
| 252 |
+
"SEPARATION_displayed_successes": "15",
|
| 253 |
+
"displayed_delta_pp_and_95CI": "-38.66 [-47.90,-28.57]",
|
| 254 |
+
"unshown_pair_delta_pp_and_95CI": "+10.92 [+5.88,+16.81]",
|
| 255 |
+
"all_legal_N": "120",
|
| 256 |
+
"all_legal_displayed_delta_pp_and_95CI": "-39.17 [-48.33,-29.17]",
|
| 257 |
+
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"model": "Qwen3-8B",
|
| 261 |
+
"base_count": "5",
|
| 262 |
+
"displayed_examples": "6",
|
| 263 |
+
"primary_family_N": "120",
|
| 264 |
+
"CONTROL_displayed_successes": "67",
|
| 265 |
+
"SEPARATION_displayed_successes": "12",
|
| 266 |
+
"displayed_delta_pp_and_95CI": "-45.83 [-55.00,-36.67]",
|
| 267 |
+
"unshown_pair_delta_pp_and_95CI": "+9.17 [+4.17,+15.00]",
|
| 268 |
+
"all_legal_N": "120",
|
| 269 |
+
"all_legal_displayed_delta_pp_and_95CI": "-45.83 [-55.00,-36.67]",
|
| 270 |
+
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"model": "Qwen3-8B",
|
| 274 |
+
"base_count": "6",
|
| 275 |
+
"displayed_examples": "7",
|
| 276 |
+
"primary_family_N": "120",
|
| 277 |
+
"CONTROL_displayed_successes": "68",
|
| 278 |
+
"SEPARATION_displayed_successes": "19",
|
| 279 |
+
"displayed_delta_pp_and_95CI": "-40.83 [-50.00,-31.67]",
|
| 280 |
+
"unshown_pair_delta_pp_and_95CI": "+12.50 [+6.67,+18.33]",
|
| 281 |
+
"all_legal_N": "120",
|
| 282 |
+
"all_legal_displayed_delta_pp_and_95CI": "-40.83 [-50.00,-31.67]",
|
| 283 |
+
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
|
| 284 |
+
},
|
| 285 |
+
{
|
| 286 |
+
"model": "Olmo-3-7B-Think",
|
| 287 |
+
"base_count": "3",
|
| 288 |
+
"displayed_examples": "4",
|
| 289 |
+
"primary_family_N": "119",
|
| 290 |
+
"CONTROL_displayed_successes": "53",
|
| 291 |
+
"SEPARATION_displayed_successes": "14",
|
| 292 |
+
"displayed_delta_pp_and_95CI": "-32.77 [-42.86,-22.69]",
|
| 293 |
+
"unshown_pair_delta_pp_and_95CI": "+8.40 [+3.36,+13.45]",
|
| 294 |
+
"all_legal_N": "120",
|
| 295 |
+
"all_legal_displayed_delta_pp_and_95CI": "-33.33 [-43.33,-23.33]",
|
| 296 |
+
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
|
| 297 |
+
},
|
| 298 |
+
{
|
| 299 |
+
"model": "Olmo-3-7B-Think",
|
| 300 |
+
"base_count": "4",
|
| 301 |
+
"displayed_examples": "5",
|
| 302 |
+
"primary_family_N": "119",
|
| 303 |
+
"CONTROL_displayed_successes": "47",
|
| 304 |
+
"SEPARATION_displayed_successes": "19",
|
| 305 |
+
"displayed_delta_pp_and_95CI": "-23.53 [-34.45,-12.61]",
|
| 306 |
+
"unshown_pair_delta_pp_and_95CI": "+11.76 [+5.88,+17.65]",
|
| 307 |
+
"all_legal_N": "120",
|
| 308 |
+
"all_legal_displayed_delta_pp_and_95CI": "-23.33 [-34.17,-12.50]",
|
| 309 |
+
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
|
| 310 |
+
},
|
| 311 |
+
{
|
| 312 |
+
"model": "Olmo-3-7B-Think",
|
| 313 |
+
"base_count": "5",
|
| 314 |
+
"displayed_examples": "6",
|
| 315 |
+
"primary_family_N": "120",
|
| 316 |
+
"CONTROL_displayed_successes": "52",
|
| 317 |
+
"SEPARATION_displayed_successes": "15",
|
| 318 |
+
"displayed_delta_pp_and_95CI": "-30.83 [-40.83,-20.83]",
|
| 319 |
+
"unshown_pair_delta_pp_and_95CI": "+11.67 [+6.67,+17.50]",
|
| 320 |
+
"all_legal_N": "120",
|
| 321 |
+
"all_legal_displayed_delta_pp_and_95CI": "-30.83 [-40.83,-20.83]",
|
| 322 |
+
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
|
| 323 |
+
},
|
| 324 |
+
{
|
| 325 |
+
"model": "Olmo-3-7B-Think",
|
| 326 |
+
"base_count": "6",
|
| 327 |
+
"displayed_examples": "7",
|
| 328 |
+
"primary_family_N": "120",
|
| 329 |
+
"CONTROL_displayed_successes": "54",
|
| 330 |
+
"SEPARATION_displayed_successes": "13",
|
| 331 |
+
"displayed_delta_pp_and_95CI": "-34.17 [-43.33,-25.00]",
|
| 332 |
+
"unshown_pair_delta_pp_and_95CI": "+9.17 [+4.17,+15.00]",
|
| 333 |
+
"all_legal_N": "120",
|
| 334 |
+
"all_legal_displayed_delta_pp_and_95CI": "-34.17 [-43.33,-25.00]",
|
| 335 |
+
"interval_scope": "POINTWISE_DESCRIPTIVE_FAMILY_PAIRED"
|
| 336 |
+
}
|
| 337 |
+
]
|
| 338 |
+
}
|
fixtures/end_to_end.json
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"purpose": "End-to-end replay of the existing Figure 1 family, selected by its role in the paper; uses both models and arms at base five, without choosing outputs by success.",
|
| 3 |
+
"sample_id": "3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e",
|
| 4 |
+
"base_count": 5,
|
| 5 |
+
"reference_rule_pairs": [
|
| 6 |
+
[
|
| 7 |
+
"ex",
|
| 8 |
+
"azw"
|
| 9 |
+
],
|
| 10 |
+
[
|
| 11 |
+
"wa",
|
| 12 |
+
"y"
|
| 13 |
+
]
|
| 14 |
+
],
|
| 15 |
+
"task_pair": {
|
| 16 |
+
"CONTROL": {
|
| 17 |
+
"inputs": [
|
| 18 |
+
"gud",
|
| 19 |
+
"czua",
|
| 20 |
+
"bwcied",
|
| 21 |
+
"dzexa",
|
| 22 |
+
"yf",
|
| 23 |
+
"iiiexa"
|
| 24 |
+
],
|
| 25 |
+
"outputs": [
|
| 26 |
+
"gud",
|
| 27 |
+
"czua",
|
| 28 |
+
"bwcied",
|
| 29 |
+
"dzazy",
|
| 30 |
+
"yf",
|
| 31 |
+
"iiiazy"
|
| 32 |
+
]
|
| 33 |
+
},
|
| 34 |
+
"SEPARATION": {
|
| 35 |
+
"inputs": [
|
| 36 |
+
"gud",
|
| 37 |
+
"czua",
|
| 38 |
+
"bwcied",
|
| 39 |
+
"dzexa",
|
| 40 |
+
"yf",
|
| 41 |
+
"wbawex"
|
| 42 |
+
],
|
| 43 |
+
"outputs": [
|
| 44 |
+
"gud",
|
| 45 |
+
"czua",
|
| 46 |
+
"bwcied",
|
| 47 |
+
"dzazy",
|
| 48 |
+
"yf",
|
| 49 |
+
"wbayzw"
|
| 50 |
+
]
|
| 51 |
+
}
|
| 52 |
+
},
|
| 53 |
+
"request_ids": [
|
| 54 |
+
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL",
|
| 55 |
+
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION",
|
| 56 |
+
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL",
|
| 57 |
+
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION"
|
| 58 |
+
],
|
| 59 |
+
"expected": {
|
| 60 |
+
"status": "PASS",
|
| 61 |
+
"family": "3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e",
|
| 62 |
+
"base_count": 5,
|
| 63 |
+
"certificates": {
|
| 64 |
+
"CONTROL": {
|
| 65 |
+
"complete_one_rule_solutions": [
|
| 66 |
+
[
|
| 67 |
+
"exa",
|
| 68 |
+
"azy"
|
| 69 |
+
]
|
| 70 |
+
],
|
| 71 |
+
"minimum_length": 1,
|
| 72 |
+
"two_rule_reference_passes": true
|
| 73 |
+
},
|
| 74 |
+
"SEPARATION": {
|
| 75 |
+
"complete_one_rule_solutions": [],
|
| 76 |
+
"minimum_length": 2,
|
| 77 |
+
"two_rule_reference_passes": true
|
| 78 |
+
}
|
| 79 |
+
},
|
| 80 |
+
"saved_outputs": {
|
| 81 |
+
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION": {
|
| 82 |
+
"selected_rules": null,
|
| 83 |
+
"legal_selected_program": false,
|
| 84 |
+
"targets": {
|
| 85 |
+
"displayed": {
|
| 86 |
+
"hits": 0,
|
| 87 |
+
"inputs": 6,
|
| 88 |
+
"all_pass": false
|
| 89 |
+
},
|
| 90 |
+
"common_nine": {
|
| 91 |
+
"hits": 0,
|
| 92 |
+
"inputs": 9,
|
| 93 |
+
"all_pass": false
|
| 94 |
+
},
|
| 95 |
+
"wholly_unshown_pair": {
|
| 96 |
+
"hits": 0,
|
| 97 |
+
"inputs": 2,
|
| 98 |
+
"all_pass": false
|
| 99 |
+
},
|
| 100 |
+
"uniform": {
|
| 101 |
+
"hits": 0,
|
| 102 |
+
"inputs": 32,
|
| 103 |
+
"all_pass": false
|
| 104 |
+
},
|
| 105 |
+
"active": {
|
| 106 |
+
"hits": 0,
|
| 107 |
+
"inputs": 32,
|
| 108 |
+
"all_pass": false
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
},
|
| 112 |
+
"M1|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL": {
|
| 113 |
+
"selected_rules": [
|
| 114 |
+
[
|
| 115 |
+
"a",
|
| 116 |
+
"y"
|
| 117 |
+
],
|
| 118 |
+
[
|
| 119 |
+
"e",
|
| 120 |
+
"a"
|
| 121 |
+
],
|
| 122 |
+
[
|
| 123 |
+
"x",
|
| 124 |
+
"z"
|
| 125 |
+
]
|
| 126 |
+
],
|
| 127 |
+
"legal_selected_program": true,
|
| 128 |
+
"targets": {
|
| 129 |
+
"displayed": {
|
| 130 |
+
"hits": 4,
|
| 131 |
+
"inputs": 6,
|
| 132 |
+
"all_pass": false
|
| 133 |
+
},
|
| 134 |
+
"common_nine": {
|
| 135 |
+
"hits": 5,
|
| 136 |
+
"inputs": 9,
|
| 137 |
+
"all_pass": false
|
| 138 |
+
},
|
| 139 |
+
"wholly_unshown_pair": {
|
| 140 |
+
"hits": 1,
|
| 141 |
+
"inputs": 2,
|
| 142 |
+
"all_pass": false
|
| 143 |
+
},
|
| 144 |
+
"uniform": {
|
| 145 |
+
"hits": 11,
|
| 146 |
+
"inputs": 32,
|
| 147 |
+
"all_pass": false
|
| 148 |
+
},
|
| 149 |
+
"active": {
|
| 150 |
+
"hits": 5,
|
| 151 |
+
"inputs": 32,
|
| 152 |
+
"all_pass": false
|
| 153 |
+
}
|
| 154 |
+
}
|
| 155 |
+
},
|
| 156 |
+
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|SEPARATION": {
|
| 157 |
+
"selected_rules": [
|
| 158 |
+
[
|
| 159 |
+
"exa",
|
| 160 |
+
"azy"
|
| 161 |
+
],
|
| 162 |
+
[
|
| 163 |
+
"wex",
|
| 164 |
+
"yzw"
|
| 165 |
+
]
|
| 166 |
+
],
|
| 167 |
+
"legal_selected_program": true,
|
| 168 |
+
"targets": {
|
| 169 |
+
"displayed": {
|
| 170 |
+
"hits": 6,
|
| 171 |
+
"inputs": 6,
|
| 172 |
+
"all_pass": true
|
| 173 |
+
},
|
| 174 |
+
"common_nine": {
|
| 175 |
+
"hits": 9,
|
| 176 |
+
"inputs": 9,
|
| 177 |
+
"all_pass": true
|
| 178 |
+
},
|
| 179 |
+
"wholly_unshown_pair": {
|
| 180 |
+
"hits": 2,
|
| 181 |
+
"inputs": 2,
|
| 182 |
+
"all_pass": true
|
| 183 |
+
},
|
| 184 |
+
"uniform": {
|
| 185 |
+
"hits": 32,
|
| 186 |
+
"inputs": 32,
|
| 187 |
+
"all_pass": true
|
| 188 |
+
},
|
| 189 |
+
"active": {
|
| 190 |
+
"hits": 0,
|
| 191 |
+
"inputs": 32,
|
| 192 |
+
"all_pass": false
|
| 193 |
+
}
|
| 194 |
+
}
|
| 195 |
+
},
|
| 196 |
+
"M2|3df6eadc4dcae8679ce01f44a61b27e1f3c0e18af27873213d79a240a74d904e|B5|CONTROL": {
|
| 197 |
+
"selected_rules": null,
|
| 198 |
+
"legal_selected_program": false,
|
| 199 |
+
"targets": {
|
| 200 |
+
"displayed": {
|
| 201 |
+
"hits": 0,
|
| 202 |
+
"inputs": 6,
|
| 203 |
+
"all_pass": false
|
| 204 |
+
},
|
| 205 |
+
"common_nine": {
|
| 206 |
+
"hits": 0,
|
| 207 |
+
"inputs": 9,
|
| 208 |
+
"all_pass": false
|
| 209 |
+
},
|
| 210 |
+
"wholly_unshown_pair": {
|
| 211 |
+
"hits": 0,
|
| 212 |
+
"inputs": 2,
|
| 213 |
+
"all_pass": false
|
| 214 |
+
},
|
| 215 |
+
"uniform": {
|
| 216 |
+
"hits": 0,
|
| 217 |
+
"inputs": 32,
|
| 218 |
+
"all_pass": false
|
| 219 |
+
},
|
| 220 |
+
"active": {
|
| 221 |
+
"hits": 0,
|
| 222 |
+
"inputs": 32,
|
| 223 |
+
"all_pass": false
|
| 224 |
+
}
|
| 225 |
+
}
|
| 226 |
+
}
|
| 227 |
+
},
|
| 228 |
+
"model_inference_calls": 0
|
| 229 |
+
}
|
| 230 |
+
}
|
index.html
CHANGED
|
@@ -1,19 +1,19 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html>
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
</html>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="description" content="Change one demonstration to see when a second rule becomes necessary. Explore certificate-grounded program induction from arXiv:2611.24975."><title>Program structure certificates · Wenzhang Du</title><link rel="stylesheet" href="style.css"><script src="explore.js" defer></script></head>
|
| 3 |
+
<body><main><header class="hero"><div class="wrap"><p class="over">PROGRAM INDUCTION / arXiv 2611.24975</p><h1>When is program<br>structure required?</h1><p class="lede">Two tasks share five demonstrations. Change the sixth and the shortest program grows from one rule to two.</p><p class="author">Wenzhang Du · November 2026</p><nav><a href="https://arxiv.org/abs/2611.24975" target="_blank" rel="noopener">Paper ↗</a><a href="https://github.com/dqswordman/program-structure-certificates" target="_blank" rel="noopener">GitHub ↗</a><a href="https://dqswordman.github.io/technical-reports/program-structure-certificates/" target="_blank" rel="noopener">Technical report ↗</a><a href="https://dqswordman.github.io/zh/technical-reports/program-structure-certificates/" target="_blank" rel="noopener">中文伴读 ↗</a></nav></div></header>
|
| 4 |
+
<div class="wrap"><section><p class="over">FIGURE 1 · WORKED EXAMPLE</p><h2>Switch the last demonstration</h2><p>Both tasks share the reference program <code>ex → azw</code> followed by <code>wa → y</code>. Try the single-rule explanation <code>exa → azy</code> on each last input.</p>
|
| 5 |
+
<fieldset id="arms" hidden><legend>Demonstration</legend><label><input type="radio" name="arm" value="CONTROL" checked>Control</label><label><input type="radio" name="arm" value="SEPARATION">Separation</label></fieldset>
|
| 6 |
+
<div class="case" id="case" aria-live="polite"><div><small>Last input</small><strong id="input">iiiexa</strong></div><div><small>Required output</small><strong id="target">iiiazy</strong></div><div><small>Shortcut output</small><strong id="shortcut">iiiazy</strong></div></div>
|
| 7 |
+
<p id="verdict" role="status">Control · one legal single-rule solution meets all six demonstrations. Certified minimum length 1.</p>
|
| 8 |
+
<details><summary>See the five shared demonstrations</summary><pre>gud → gud
|
| 9 |
+
czua → czua
|
| 10 |
+
bwcied → bwcied
|
| 11 |
+
dzexa → dzazy
|
| 12 |
+
yf → yf</pre></details></section>
|
| 13 |
+
<section><p class="over">ARCHIVED MODEL OUTCOMES</p><h2>Two ways to score a program</h2><p>Each column uses the same certified families. Effects are separation minus control in percentage points, with pointwise 95% paired intervals.</p>
|
| 14 |
+
<div class="controls" id="models" hidden><label>Model<select id="model"><option>Qwen3-8B</option><option>Olmo-3-7B-Think</option></select></label><label>Shared examples<select id="base"><option>3</option><option>4</option><option selected>5</option><option>6</option></select></label></div>
|
| 15 |
+
<div class="results" aria-live="polite"><p id="counts">120 families · 67 control and 12 separation tasks completed.</p><dl><div><dt>Displayed task Δ</dt><dd id="displayed">−45.83 [−55.00, −36.67]</dd></div><div><dt>Hidden pair Δ</dt><dd id="hidden-pair">+9.17 [+4.17, +15.00]</dd></div></dl></div>
|
| 16 |
+
<p class="note">The hidden pair probes a particular distinction. Across all eight settings, separation programs agree with the reference on only 1.93–5.78% of inputs activating both rules.</p></section>
|
| 17 |
+
<section><h2>Replay on CPU</h2><p>The offline edition includes task certificates, normalized trajectories, reference tables and the bilingual companion. With NumPy installed, the full replay verifies 1,920 saved trajectories.</p><pre>python -B verify_manifest.py
|
| 18 |
+
python -B replay_fixture.py
|
| 19 |
+
python -B reproduce.py --out replay</pre><nav><a class="primary" href="https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates/resolve/main/program-structure-certificates-v1.0.0.zip" target="_blank" rel="noopener">Download v1.0.0 ZIP ↓</a><a href="https://github.com/dqswordman/program-structure-certificates/blob/main/docs/reproduction.md" target="_blank" rel="noopener">Reproduction guide ↗</a><a href="https://huggingface.co/spaces/DQSWORDMAN/program-structure-certificates/resolve/main/SNAPSHOT.json" target="_blank" rel="noopener">Checksums ↗</a></nav></section></div></main></body></html>
|
licenses/CC-BY-4.0.txt
ADDED
|
@@ -0,0 +1,395 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Attribution 4.0 International
|
| 2 |
+
|
| 3 |
+
=======================================================================
|
| 4 |
+
|
| 5 |
+
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
| 6 |
+
does not provide legal services or legal advice. Distribution of
|
| 7 |
+
Creative Commons public licenses does not create a lawyer-client or
|
| 8 |
+
other relationship. Creative Commons makes its licenses and related
|
| 9 |
+
information available on an "as-is" basis. Creative Commons gives no
|
| 10 |
+
warranties regarding its licenses, any material licensed under their
|
| 11 |
+
terms and conditions, or any related information. Creative Commons
|
| 12 |
+
disclaims all liability for damages resulting from their use to the
|
| 13 |
+
fullest extent possible.
|
| 14 |
+
|
| 15 |
+
Using Creative Commons Public Licenses
|
| 16 |
+
|
| 17 |
+
Creative Commons public licenses provide a standard set of terms and
|
| 18 |
+
conditions that creators and other rights holders may use to share
|
| 19 |
+
original works of authorship and other material subject to copyright
|
| 20 |
+
and certain other rights specified in the public license below. The
|
| 21 |
+
following considerations are for informational purposes only, are not
|
| 22 |
+
exhaustive, and do not form part of our licenses.
|
| 23 |
+
|
| 24 |
+
Considerations for licensors: Our public licenses are
|
| 25 |
+
intended for use by those authorized to give the public
|
| 26 |
+
permission to use material in ways otherwise restricted by
|
| 27 |
+
copyright and certain other rights. Our licenses are
|
| 28 |
+
irrevocable. Licensors should read and understand the terms
|
| 29 |
+
and conditions of the license they choose before applying it.
|
| 30 |
+
Licensors should also secure all rights necessary before
|
| 31 |
+
applying our licenses so that the public can reuse the
|
| 32 |
+
material as expected. Licensors should clearly mark any
|
| 33 |
+
material not subject to the license. This includes other CC-
|
| 34 |
+
licensed material, or material used under an exception or
|
| 35 |
+
limitation to copyright. More considerations for licensors:
|
| 36 |
+
wiki.creativecommons.org/Considerations_for_licensors
|
| 37 |
+
|
| 38 |
+
Considerations for the public: By using one of our public
|
| 39 |
+
licenses, a licensor grants the public permission to use the
|
| 40 |
+
licensed material under specified terms and conditions. If
|
| 41 |
+
the licensor's permission is not necessary for any reason--for
|
| 42 |
+
example, because of any applicable exception or limitation to
|
| 43 |
+
copyright--then that use is not regulated by the license. Our
|
| 44 |
+
licenses grant only permissions under copyright and certain
|
| 45 |
+
other rights that a licensor has authority to grant. Use of
|
| 46 |
+
the licensed material may still be restricted for other
|
| 47 |
+
reasons, including because others have copyright or other
|
| 48 |
+
rights in the material. A licensor may make special requests,
|
| 49 |
+
such as asking that all changes be marked or described.
|
| 50 |
+
Although not required by our licenses, you are encouraged to
|
| 51 |
+
respect those requests where reasonable. More considerations
|
| 52 |
+
for the public:
|
| 53 |
+
wiki.creativecommons.org/Considerations_for_licensees
|
| 54 |
+
|
| 55 |
+
=======================================================================
|
| 56 |
+
|
| 57 |
+
Creative Commons Attribution 4.0 International Public License
|
| 58 |
+
|
| 59 |
+
By exercising the Licensed Rights (defined below), You accept and agree
|
| 60 |
+
to be bound by the terms and conditions of this Creative Commons
|
| 61 |
+
Attribution 4.0 International Public License ("Public License"). To the
|
| 62 |
+
extent this Public License may be interpreted as a contract, You are
|
| 63 |
+
granted the Licensed Rights in consideration of Your acceptance of
|
| 64 |
+
these terms and conditions, and the Licensor grants You such rights in
|
| 65 |
+
consideration of benefits the Licensor receives from making the
|
| 66 |
+
Licensed Material available under these terms and conditions.
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
Section 1 -- Definitions.
|
| 70 |
+
|
| 71 |
+
a. Adapted Material means material subject to Copyright and Similar
|
| 72 |
+
Rights that is derived from or based upon the Licensed Material
|
| 73 |
+
and in which the Licensed Material is translated, altered,
|
| 74 |
+
arranged, transformed, or otherwise modified in a manner requiring
|
| 75 |
+
permission under the Copyright and Similar Rights held by the
|
| 76 |
+
Licensor. For purposes of this Public License, where the Licensed
|
| 77 |
+
Material is a musical work, performance, or sound recording,
|
| 78 |
+
Adapted Material is always produced where the Licensed Material is
|
| 79 |
+
synched in timed relation with a moving image.
|
| 80 |
+
|
| 81 |
+
b. Adapter's License means the license You apply to Your Copyright
|
| 82 |
+
and Similar Rights in Your contributions to Adapted Material in
|
| 83 |
+
accordance with the terms and conditions of this Public License.
|
| 84 |
+
|
| 85 |
+
c. Copyright and Similar Rights means copyright and/or similar rights
|
| 86 |
+
closely related to copyright including, without limitation,
|
| 87 |
+
performance, broadcast, sound recording, and Sui Generis Database
|
| 88 |
+
Rights, without regard to how the rights are labeled or
|
| 89 |
+
categorized. For purposes of this Public License, the rights
|
| 90 |
+
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
| 91 |
+
Rights.
|
| 92 |
+
|
| 93 |
+
d. Effective Technological Measures means those measures that, in the
|
| 94 |
+
absence of proper authority, may not be circumvented under laws
|
| 95 |
+
fulfilling obligations under Article 11 of the WIPO Copyright
|
| 96 |
+
Treaty adopted on December 20, 1996, and/or similar international
|
| 97 |
+
agreements.
|
| 98 |
+
|
| 99 |
+
e. Exceptions and Limitations means fair use, fair dealing, and/or
|
| 100 |
+
any other exception or limitation to Copyright and Similar Rights
|
| 101 |
+
that applies to Your use of the Licensed Material.
|
| 102 |
+
|
| 103 |
+
f. Licensed Material means the artistic or literary work, database,
|
| 104 |
+
or other material to which the Licensor applied this Public
|
| 105 |
+
License.
|
| 106 |
+
|
| 107 |
+
g. Licensed Rights means the rights granted to You subject to the
|
| 108 |
+
terms and conditions of this Public License, which are limited to
|
| 109 |
+
all Copyright and Similar Rights that apply to Your use of the
|
| 110 |
+
Licensed Material and that the Licensor has authority to license.
|
| 111 |
+
|
| 112 |
+
h. Licensor means the individual(s) or entity(ies) granting rights
|
| 113 |
+
under this Public License.
|
| 114 |
+
|
| 115 |
+
i. Share means to provide material to the public by any means or
|
| 116 |
+
process that requires permission under the Licensed Rights, such
|
| 117 |
+
as reproduction, public display, public performance, distribution,
|
| 118 |
+
dissemination, communication, or importation, and to make material
|
| 119 |
+
available to the public including in ways that members of the
|
| 120 |
+
public may access the material from a place and at a time
|
| 121 |
+
individually chosen by them.
|
| 122 |
+
|
| 123 |
+
j. Sui Generis Database Rights means rights other than copyright
|
| 124 |
+
resulting from Directive 96/9/EC of the European Parliament and of
|
| 125 |
+
the Council of 11 March 1996 on the legal protection of databases,
|
| 126 |
+
as amended and/or succeeded, as well as other essentially
|
| 127 |
+
equivalent rights anywhere in the world.
|
| 128 |
+
|
| 129 |
+
k. You means the individual or entity exercising the Licensed Rights
|
| 130 |
+
under this Public License. Your has a corresponding meaning.
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
Section 2 -- Scope.
|
| 134 |
+
|
| 135 |
+
a. License grant.
|
| 136 |
+
|
| 137 |
+
1. Subject to the terms and conditions of this Public License,
|
| 138 |
+
the Licensor hereby grants You a worldwide, royalty-free,
|
| 139 |
+
non-sublicensable, non-exclusive, irrevocable license to
|
| 140 |
+
exercise the Licensed Rights in the Licensed Material to:
|
| 141 |
+
|
| 142 |
+
a. reproduce and Share the Licensed Material, in whole or
|
| 143 |
+
in part; and
|
| 144 |
+
|
| 145 |
+
b. produce, reproduce, and Share Adapted Material.
|
| 146 |
+
|
| 147 |
+
2. Exceptions and Limitations. For the avoidance of doubt, where
|
| 148 |
+
Exceptions and Limitations apply to Your use, this Public
|
| 149 |
+
License does not apply, and You do not need to comply with
|
| 150 |
+
its terms and conditions.
|
| 151 |
+
|
| 152 |
+
3. Term. The term of this Public License is specified in Section
|
| 153 |
+
6(a).
|
| 154 |
+
|
| 155 |
+
4. Media and formats; technical modifications allowed. The
|
| 156 |
+
Licensor authorizes You to exercise the Licensed Rights in
|
| 157 |
+
all media and formats whether now known or hereafter created,
|
| 158 |
+
and to make technical modifications necessary to do so. The
|
| 159 |
+
Licensor waives and/or agrees not to assert any right or
|
| 160 |
+
authority to forbid You from making technical modifications
|
| 161 |
+
necessary to exercise the Licensed Rights, including
|
| 162 |
+
technical modifications necessary to circumvent Effective
|
| 163 |
+
Technological Measures. For purposes of this Public License,
|
| 164 |
+
simply making modifications authorized by this Section 2(a)
|
| 165 |
+
(4) never produces Adapted Material.
|
| 166 |
+
|
| 167 |
+
5. Downstream recipients.
|
| 168 |
+
|
| 169 |
+
a. Offer from the Licensor -- Licensed Material. Every
|
| 170 |
+
recipient of the Licensed Material automatically
|
| 171 |
+
receives an offer from the Licensor to exercise the
|
| 172 |
+
Licensed Rights under the terms and conditions of this
|
| 173 |
+
Public License.
|
| 174 |
+
|
| 175 |
+
b. No downstream restrictions. You may not offer or impose
|
| 176 |
+
any additional or different terms or conditions on, or
|
| 177 |
+
apply any Effective Technological Measures to, the
|
| 178 |
+
Licensed Material if doing so restricts exercise of the
|
| 179 |
+
Licensed Rights by any recipient of the Licensed
|
| 180 |
+
Material.
|
| 181 |
+
|
| 182 |
+
6. No endorsement. Nothing in this Public License constitutes or
|
| 183 |
+
may be construed as permission to assert or imply that You
|
| 184 |
+
are, or that Your use of the Licensed Material is, connected
|
| 185 |
+
with, or sponsored, endorsed, or granted official status by,
|
| 186 |
+
the Licensor or others designated to receive attribution as
|
| 187 |
+
provided in Section 3(a)(1)(A)(i).
|
| 188 |
+
|
| 189 |
+
b. Other rights.
|
| 190 |
+
|
| 191 |
+
1. Moral rights, such as the right of integrity, are not
|
| 192 |
+
licensed under this Public License, nor are publicity,
|
| 193 |
+
privacy, and/or other similar personality rights; however, to
|
| 194 |
+
the extent possible, the Licensor waives and/or agrees not to
|
| 195 |
+
assert any such rights held by the Licensor to the limited
|
| 196 |
+
extent necessary to allow You to exercise the Licensed
|
| 197 |
+
Rights, but not otherwise.
|
| 198 |
+
|
| 199 |
+
2. Patent and trademark rights are not licensed under this
|
| 200 |
+
Public License.
|
| 201 |
+
|
| 202 |
+
3. To the extent possible, the Licensor waives any right to
|
| 203 |
+
collect royalties from You for the exercise of the Licensed
|
| 204 |
+
Rights, whether directly or through a collecting society
|
| 205 |
+
under any voluntary or waivable statutory or compulsory
|
| 206 |
+
licensing scheme. In all other cases the Licensor expressly
|
| 207 |
+
reserves any right to collect such royalties.
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
Section 3 -- License Conditions.
|
| 211 |
+
|
| 212 |
+
Your exercise of the Licensed Rights is expressly made subject to the
|
| 213 |
+
following conditions.
|
| 214 |
+
|
| 215 |
+
a. Attribution.
|
| 216 |
+
|
| 217 |
+
1. If You Share the Licensed Material (including in modified
|
| 218 |
+
form), You must:
|
| 219 |
+
|
| 220 |
+
a. retain the following if it is supplied by the Licensor
|
| 221 |
+
with the Licensed Material:
|
| 222 |
+
|
| 223 |
+
i. identification of the creator(s) of the Licensed
|
| 224 |
+
Material and any others designated to receive
|
| 225 |
+
attribution, in any reasonable manner requested by
|
| 226 |
+
the Licensor (including by pseudonym if
|
| 227 |
+
designated);
|
| 228 |
+
|
| 229 |
+
ii. a copyright notice;
|
| 230 |
+
|
| 231 |
+
iii. a notice that refers to this Public License;
|
| 232 |
+
|
| 233 |
+
iv. a notice that refers to the disclaimer of
|
| 234 |
+
warranties;
|
| 235 |
+
|
| 236 |
+
v. a URI or hyperlink to the Licensed Material to the
|
| 237 |
+
extent reasonably practicable;
|
| 238 |
+
|
| 239 |
+
b. indicate if You modified the Licensed Material and
|
| 240 |
+
retain an indication of any previous modifications; and
|
| 241 |
+
|
| 242 |
+
c. indicate the Licensed Material is licensed under this
|
| 243 |
+
Public License, and include the text of, or the URI or
|
| 244 |
+
hyperlink to, this Public License.
|
| 245 |
+
|
| 246 |
+
2. You may satisfy the conditions in Section 3(a)(1) in any
|
| 247 |
+
reasonable manner based on the medium, means, and context in
|
| 248 |
+
which You Share the Licensed Material. For example, it may be
|
| 249 |
+
reasonable to satisfy the conditions by providing a URI or
|
| 250 |
+
hyperlink to a resource that includes the required
|
| 251 |
+
information.
|
| 252 |
+
|
| 253 |
+
3. If requested by the Licensor, You must remove any of the
|
| 254 |
+
information required by Section 3(a)(1)(A) to the extent
|
| 255 |
+
reasonably practicable.
|
| 256 |
+
|
| 257 |
+
4. If You Share Adapted Material You produce, the Adapter's
|
| 258 |
+
License You apply must not prevent recipients of the Adapted
|
| 259 |
+
Material from complying with this Public License.
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
Section 4 -- Sui Generis Database Rights.
|
| 263 |
+
|
| 264 |
+
Where the Licensed Rights include Sui Generis Database Rights that
|
| 265 |
+
apply to Your use of the Licensed Material:
|
| 266 |
+
|
| 267 |
+
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
| 268 |
+
to extract, reuse, reproduce, and Share all or a substantial
|
| 269 |
+
portion of the contents of the database;
|
| 270 |
+
|
| 271 |
+
b. if You include all or a substantial portion of the database
|
| 272 |
+
contents in a database in which You have Sui Generis Database
|
| 273 |
+
Rights, then the database in which You have Sui Generis Database
|
| 274 |
+
Rights (but not its individual contents) is Adapted Material; and
|
| 275 |
+
|
| 276 |
+
c. You must comply with the conditions in Section 3(a) if You Share
|
| 277 |
+
all or a substantial portion of the contents of the database.
|
| 278 |
+
|
| 279 |
+
For the avoidance of doubt, this Section 4 supplements and does not
|
| 280 |
+
replace Your obligations under this Public License where the Licensed
|
| 281 |
+
Rights include other Copyright and Similar Rights.
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
| 285 |
+
|
| 286 |
+
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
| 287 |
+
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
| 288 |
+
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
| 289 |
+
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
| 290 |
+
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
| 291 |
+
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
| 292 |
+
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
| 293 |
+
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
| 294 |
+
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
| 295 |
+
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
| 296 |
+
|
| 297 |
+
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
| 298 |
+
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
| 299 |
+
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
| 300 |
+
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
| 301 |
+
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
| 302 |
+
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
| 303 |
+
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
| 304 |
+
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
| 305 |
+
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
| 306 |
+
|
| 307 |
+
c. The disclaimer of warranties and limitation of liability provided
|
| 308 |
+
above shall be interpreted in a manner that, to the extent
|
| 309 |
+
possible, most closely approximates an absolute disclaimer and
|
| 310 |
+
waiver of all liability.
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
Section 6 -- Term and Termination.
|
| 314 |
+
|
| 315 |
+
a. This Public License applies for the term of the Copyright and
|
| 316 |
+
Similar Rights licensed here. However, if You fail to comply with
|
| 317 |
+
this Public License, then Your rights under this Public License
|
| 318 |
+
terminate automatically.
|
| 319 |
+
|
| 320 |
+
b. Where Your right to use the Licensed Material has terminated under
|
| 321 |
+
Section 6(a), it reinstates:
|
| 322 |
+
|
| 323 |
+
1. automatically as of the date the violation is cured, provided
|
| 324 |
+
it is cured within 30 days of Your discovery of the
|
| 325 |
+
violation; or
|
| 326 |
+
|
| 327 |
+
2. upon express reinstatement by the Licensor.
|
| 328 |
+
|
| 329 |
+
For the avoidance of doubt, this Section 6(b) does not affect any
|
| 330 |
+
right the Licensor may have to seek remedies for Your violations
|
| 331 |
+
of this Public License.
|
| 332 |
+
|
| 333 |
+
c. For the avoidance of doubt, the Licensor may also offer the
|
| 334 |
+
Licensed Material under separate terms or conditions or stop
|
| 335 |
+
distributing the Licensed Material at any time; however, doing so
|
| 336 |
+
will not terminate this Public License.
|
| 337 |
+
|
| 338 |
+
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
| 339 |
+
License.
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
Section 7 -- Other Terms and Conditions.
|
| 343 |
+
|
| 344 |
+
a. The Licensor shall not be bound by any additional or different
|
| 345 |
+
terms or conditions communicated by You unless expressly agreed.
|
| 346 |
+
|
| 347 |
+
b. Any arrangements, understandings, or agreements regarding the
|
| 348 |
+
Licensed Material not stated herein are separate from and
|
| 349 |
+
independent of the terms and conditions of this Public License.
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
Section 8 -- Interpretation.
|
| 353 |
+
|
| 354 |
+
a. For the avoidance of doubt, this Public License does not, and
|
| 355 |
+
shall not be interpreted to, reduce, limit, restrict, or impose
|
| 356 |
+
conditions on any use of the Licensed Material that could lawfully
|
| 357 |
+
be made without permission under this Public License.
|
| 358 |
+
|
| 359 |
+
b. To the extent possible, if any provision of this Public License is
|
| 360 |
+
deemed unenforceable, it shall be automatically reformed to the
|
| 361 |
+
minimum extent necessary to make it enforceable. If the provision
|
| 362 |
+
cannot be reformed, it shall be severed from this Public License
|
| 363 |
+
without affecting the enforceability of the remaining terms and
|
| 364 |
+
conditions.
|
| 365 |
+
|
| 366 |
+
c. No term or condition of this Public License will be waived and no
|
| 367 |
+
failure to comply consented to unless expressly agreed to by the
|
| 368 |
+
Licensor.
|
| 369 |
+
|
| 370 |
+
d. Nothing in this Public License constitutes or may be interpreted
|
| 371 |
+
as a limitation upon, or waiver of, any privileges and immunities
|
| 372 |
+
that apply to the Licensor or You, including from the legal
|
| 373 |
+
processes of any jurisdiction or authority.
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
=======================================================================
|
| 377 |
+
|
| 378 |
+
Creative Commons is not a party to its public
|
| 379 |
+
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
| 380 |
+
its public licenses to material it publishes and in those instances
|
| 381 |
+
will be considered the “Licensor.” The text of the Creative Commons
|
| 382 |
+
public licenses is dedicated to the public domain under the CC0 Public
|
| 383 |
+
Domain Dedication. Except for the limited purpose of indicating that
|
| 384 |
+
material is shared under a Creative Commons public license or as
|
| 385 |
+
otherwise permitted by the Creative Commons policies published at
|
| 386 |
+
creativecommons.org/policies, Creative Commons does not authorize the
|
| 387 |
+
use of the trademark "Creative Commons" or any other trademark or logo
|
| 388 |
+
of Creative Commons without its prior written consent including,
|
| 389 |
+
without limitation, in connection with any unauthorized modifications
|
| 390 |
+
to any of its public licenses or any other arrangements,
|
| 391 |
+
understandings, or agreements concerning use of licensed material. For
|
| 392 |
+
the avoidance of doubt, this paragraph does not form part of the
|
| 393 |
+
public licenses.
|
| 394 |
+
|
| 395 |
+
Creative Commons may be contacted at creativecommons.org.
|
licenses/PBEBench-MIT.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 LLab at CMU
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
program-structure-certificates-v1.0.0.zip
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e68af75e393e34668b2d196ea2659261f428dbaf1ad8e5556a7cec1288dc4515
|
| 3 |
+
size 27396663
|
provenance/ANALYSIS_HISTORY.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"record_type": "retained_historical_analysis_provenance",
|
| 3 |
+
"source": "v3 paper_source/appendix.tex, former Appendix E",
|
| 4 |
+
"source_sha256": "1bd3383fc790bba1f19176a180a4061e9f28659bf6f98689143f16bc45b22e91",
|
| 5 |
+
"historical_appendix_E_tex": "\\section{Statistical Analysis and Prespecified Criteria}\\label{app:prospective}\n\\subsection{Effect magnitude and statistical interpretation}\nWrite $\\widehat\\Delta=\\widehat\\theta_{\\Vis}$ and $\\widehat\\Gamma=\\widehat\\theta_{\\Com}$. Before inference, the analysis plan set a ten-percentage-point magnitude benchmark. It combined $\\widehat\\Delta\\leq-.10$ with a six-comparison upper interval endpoint below zero for completion cost, and $\\widehat\\Gamma\\geq+.10$ with a lower endpoint above zero for common-target benefit. A joint magnitude interpretation required both conditions in at least two models. Models with fewer than 12 visible successes over 240 outputs were designated support-limited.\n\nQwen and Olmo show common-nine changes of 7.50 and 9.17 percentage points, with adjusted intervals $[1.67,14.17]$ and $[3.33,16.67]$. Both intervals exclude zero and span the predeclared ten-point benchmark. Both point estimates fall below that magnitude criterion. We evaluate the $H_2$ target with its specified three-model comparison family.\n\nMarginal 95\\% intervals for visible/common differences are $[-55.00,-37.50]$/$[3.33,12.50]$ for Qwen, $[-40.83,-20.83]$/$[4.17,15.00]$ for Olmo, and $[-6.67,1.67]$/$[0,2.50]$ for corrected Phi. The main primary table retains the original six-comparison intervals.\n\n\\subsection{Exact paired-binary sensitivity}\nWe also compute exact paired-binary tests for the six original contrasts. Let $g$ and $\\ell$ count separation-only and control-only successes. Under independent paired sampling and paired marginal symmetry, $g\\mid(g+\\ell)$ is $\\operatorname{Binomial}(g+\\ell,1/2)$. We use a two-sided exact tail test and $\\min(1,6p)$ \\citep{mcnemar1947,dror2018}. These tests complement the planned percentile bootstrap.\n\n\\begin{table}[htbp]\n\\centering\\smalltable{tables/exact.tex}\n\\caption{Exact paired-binary tests for the six primary contrasts. Gain and loss denote separation-only and control-only successes.}\n\\label{tab:exact}\n\\end{table}\n\nFor common-nine, Qwen has nine gains and zero losses, giving $p=2/2^9=.00390625$ and corrected $p=.0234375$. Olmo has eleven gains and zero losses, giving $p=.0009765625$ and corrected $p=.005859375$. Phi has one gain.\n\n\\subsection{Supplemental uncertainty and chronology}\nSupplemental evaluation designs were finalized after the original model generation and before supplemental scoring. They specify the additional-input generator, solver order, alternative selectors, and analysis targets. $H_2$ uses the archived 50,000 family resamples with quantiles $1/120,119/120$ for three approximate simultaneous intervals. Its marginal intervals are $[4.17,15.00]$, $[6.67,17.50]$, and $[-2.50,2.50]$ points for Qwen, Olmo, and Phi. The primary six-comparison and supplemental three-comparison families each use their respective adjustments.\n\nThe remaining supplemental tables and curves report descriptive estimates or marginal intervals. Equal-token and fixed token-difference strata use recorded subset family IDs and saved 50,000-resample arrays. Strata with $n<12$ are reported as descriptive diagnostics. These analyses execute the saved original model outputs.\n\n",
|
| 6 |
+
"original_joint_magnitude_rule": {
|
| 7 |
+
"specified_before": "original independent validation model generation",
|
| 8 |
+
"completion_point_estimate_max": -0.1,
|
| 9 |
+
"completion_adjusted_interval_upper_endpoint": "below zero",
|
| 10 |
+
"common_target_point_estimate_min": 0.1,
|
| 11 |
+
"common_target_adjusted_interval_lower_endpoint": "above zero",
|
| 12 |
+
"minimum_models_satisfying_both": 2,
|
| 13 |
+
"support_limited_rule": "fewer than 12 visible successes over 240 outputs",
|
| 14 |
+
"observed_common_nine_gains_pp": {
|
| 15 |
+
"Qwen": 7.5,
|
| 16 |
+
"Olmo": 9.17
|
| 17 |
+
},
|
| 18 |
+
"joint_magnitude_rule_met": false
|
| 19 |
+
},
|
| 20 |
+
"current_presentation": "Appendix E.3 preserves the original magnitude interpretation in one sentence; estimates and uncertainty remain unchanged.",
|
| 21 |
+
"record_scope": "Historical manuscript transcription; not an original preregistration or a reconstructed bootstrap-index file."
|
| 22 |
+
}
|
provenance/NORMALIZATION.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "ANONYMIZED_SCIENTIFIC_DATA",
|
| 3 |
+
"trajectory_count": 1920,
|
| 4 |
+
"public_reproduction_goal": "Recompute all 1920 per-request metric rows and all 348 paired-effect rows from archived text/programs and finite targets.",
|
| 5 |
+
"retained_scientific_fields": [
|
| 6 |
+
"input/output strings",
|
| 7 |
+
"reference rules",
|
| 8 |
+
"complete one-rule certificates",
|
| 9 |
+
"raw generated text",
|
| 10 |
+
"generated and prompt token IDs",
|
| 11 |
+
"selected final programs",
|
| 12 |
+
"model revisions",
|
| 13 |
+
"batch seeds",
|
| 14 |
+
"inference contract",
|
| 15 |
+
"family/bootstrap pairing"
|
| 16 |
+
],
|
| 17 |
+
"removed_metadata": [
|
| 18 |
+
"absolute machine paths",
|
| 19 |
+
"operator and machine labels",
|
| 20 |
+
"operational timestamps",
|
| 21 |
+
"historical review records",
|
| 22 |
+
"environment/cache paths",
|
| 23 |
+
"forward-commit file layouts"
|
| 24 |
+
],
|
| 25 |
+
"raw_byte_identity_claimed": false,
|
| 26 |
+
"original_forward_commits_reconstructed": false,
|
| 27 |
+
"original_scientific_tables_preserved": true,
|
| 28 |
+
"source_adaptation": "canonical_readouts_v1.py and the archived scientific source files are copied without changes. native_score.py extracts only the definitions used in the frozen native scoring path, verbatim, and supplies portable namespace bindings. family_statistics.py copies the frozen ci function verbatim. reproduce.py supplies portable normalized-data input/output."
|
| 29 |
+
}
|
replay_fixture.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Check the Figure 1 task pair and four saved base-five outputs using only CPU.
|
| 3 |
+
|
| 4 |
+
Run from the extracted package root: python -B replay_fixture.py
|
| 5 |
+
No model, tokenizer, network access, or new model generation is used.
|
| 6 |
+
"""
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
import gzip
|
| 9 |
+
import json
|
| 10 |
+
import sys
|
| 11 |
+
|
| 12 |
+
ROOT = Path(__file__).resolve().parent
|
| 13 |
+
sys.path.insert(0, str(ROOT / 'code'))
|
| 14 |
+
import canonical_readouts_v1 as canonical
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def load_lines(path: Path) -> list[dict]:
|
| 18 |
+
with path.open(encoding='utf-8') as handle:
|
| 19 |
+
return [json.loads(line) for line in handle if line.strip()]
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def complete_one_rules(inputs: list[str], outputs: list[str]) -> list[list[str]]:
|
| 23 |
+
"""Enumerate all solutions using the nonidentity-anchor completeness lemma."""
|
| 24 |
+
anchor = next((i for i, (x, y) in enumerate(zip(inputs, outputs)) if x != y), None)
|
| 25 |
+
if anchor is None:
|
| 26 |
+
raise ValueError('Fixture enumeration requires a nonidentity example.')
|
| 27 |
+
def pieces(text: str) -> set[str]:
|
| 28 |
+
return {text[i:i+k] for k in (1, 2, 3) for i in range(len(text)-k+1)}
|
| 29 |
+
return [list(rule) for rule in sorted(
|
| 30 |
+
(a, b) for a in pieces(inputs[anchor])
|
| 31 |
+
for b in pieces(outputs[anchor]) | {''}
|
| 32 |
+
if all(x.replace(a, b) == y for x, y in zip(inputs, outputs))
|
| 33 |
+
)]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def run_fixture() -> dict:
|
| 37 |
+
fixture = json.loads((ROOT / 'fixtures/end_to_end.json').read_text(encoding='utf-8'))
|
| 38 |
+
sid, base = fixture['sample_id'], fixture['base_count']
|
| 39 |
+
family = next(x for x in load_lines(ROOT / 'data/families.jsonl') if x['sample_id'] == sid)
|
| 40 |
+
variant = next(x for x in load_lines(ROOT / 'data/base_variants.jsonl')
|
| 41 |
+
if x['sample_id'] == sid and x['base_count'] == base)
|
| 42 |
+
if fixture['task_pair'] != {arm: {k: variant['arms'][arm][k] for k in ('inputs', 'outputs')}
|
| 43 |
+
for arm in ('CONTROL', 'SEPARATION')}:
|
| 44 |
+
raise ValueError('Fixture specification differs from the saved variant.')
|
| 45 |
+
reference = [tuple(pair) for pair in fixture['reference_rule_pairs']]
|
| 46 |
+
certificates = {}
|
| 47 |
+
for arm in ('CONTROL', 'SEPARATION'):
|
| 48 |
+
task = variant['arms'][arm]
|
| 49 |
+
solutions = complete_one_rules(task['inputs'], task['outputs'])
|
| 50 |
+
if solutions != sorted(task['one_rule_solutions']):
|
| 51 |
+
raise ValueError(f'{arm}: one-rule solutions differ from the saved certificate.')
|
| 52 |
+
if canonical.apply(task['inputs'], reference) != task['outputs']:
|
| 53 |
+
raise ValueError(f'{arm}: reference fails the specification.')
|
| 54 |
+
minimum = 1 if solutions else 2
|
| 55 |
+
if minimum != task['minimum_length']:
|
| 56 |
+
raise ValueError(f'{arm}: minimum-length certificate mismatch.')
|
| 57 |
+
certificates[arm] = {'complete_one_rule_solutions': solutions,
|
| 58 |
+
'minimum_length': minimum, 'two_rule_reference_passes': True}
|
| 59 |
+
target_rows = [x for x in load_lines(ROOT / 'data/broader_inputs.jsonl') if x['sample_id'] == sid]
|
| 60 |
+
targets = {x['stratum']: x['accepted'] for x in target_rows}
|
| 61 |
+
other = family['witness_pairs'][2-family['displayed_witness_index']]
|
| 62 |
+
common = list(zip(family['original_inputs'], family['original_outputs']))
|
| 63 |
+
common += [(pair[k]['input'], pair[k]['output'])
|
| 64 |
+
for pair in family['witness_pairs'] for k in ('control', 'separation')]
|
| 65 |
+
unshown = [(other[k]['input'], other[k]['output']) for k in ('control', 'separation')]
|
| 66 |
+
requested = set(fixture['request_ids'])
|
| 67 |
+
outputs = {}
|
| 68 |
+
with gzip.open(ROOT / 'data/trajectories.jsonl.gz', 'rt', encoding='utf-8') as handle:
|
| 69 |
+
for line in handle:
|
| 70 |
+
record = json.loads(line)
|
| 71 |
+
key = record['request_id']
|
| 72 |
+
if key not in requested:
|
| 73 |
+
continue
|
| 74 |
+
selected = canonical.final_rules(record['raw_text'])
|
| 75 |
+
normalized = [list(pair) for pair in selected] if selected is not None else None
|
| 76 |
+
if normalized != record['selected_final']['rules']:
|
| 77 |
+
raise ValueError(f'{key}: selected program mismatch.')
|
| 78 |
+
task = variant['arms'][record['arm']]
|
| 79 |
+
def hits(pairs: list[tuple[str, str]]) -> int:
|
| 80 |
+
return sum(selected is not None and canonical.apply([x], selected) == [y]
|
| 81 |
+
for x, y in pairs)
|
| 82 |
+
evaluation = {'displayed': list(zip(task['inputs'], task['outputs'])),
|
| 83 |
+
'common_nine': common, 'wholly_unshown_pair': unshown,
|
| 84 |
+
**{name.lower(): [(p['input'], p['output']) for p in pts]
|
| 85 |
+
for name, pts in targets.items()}}
|
| 86 |
+
outputs[key] = {
|
| 87 |
+
'selected_rules': normalized,
|
| 88 |
+
'legal_selected_program': selected is not None,
|
| 89 |
+
'targets': {name: {'hits': hits(pairs), 'inputs': len(pairs),
|
| 90 |
+
'all_pass': hits(pairs) == len(pairs)}
|
| 91 |
+
for name, pairs in evaluation.items()}}
|
| 92 |
+
if set(outputs) != requested:
|
| 93 |
+
raise ValueError('Missing requested saved model output.')
|
| 94 |
+
return {'status': 'PASS', 'family': sid, 'base_count': base,
|
| 95 |
+
'certificates': certificates, 'saved_outputs': outputs, 'model_inference_calls': 0}
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def main() -> int:
|
| 99 |
+
try:
|
| 100 |
+
expected = json.loads((ROOT / 'fixtures/end_to_end.json').read_text())['expected']
|
| 101 |
+
actual = run_fixture()
|
| 102 |
+
if actual != expected:
|
| 103 |
+
raise ValueError('Fixture results differ from the recorded expected output.')
|
| 104 |
+
print(json.dumps(actual, indent=2))
|
| 105 |
+
return 0
|
| 106 |
+
except (OSError, KeyError, ValueError, StopIteration) as exc:
|
| 107 |
+
print(f'Fixture verification failed: {exc}', file=sys.stderr)
|
| 108 |
+
return 1
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
if __name__ == '__main__':
|
| 112 |
+
raise SystemExit(main())
|